Repository navigation
Expand file tree
/
Copy pathpycluster_soft.h
More file actions
130 lines (124 loc) · 5.83 KB
/
Copy pathpycluster_soft.h
File metadata and controls
130 lines (124 loc) · 5.83 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
#ifndef PYSCLUSTER_HEADER_H__
#define PYSCLUSTER_HEADER_H__
#include "pycluster.h"
#include "pybind11/embed.h"
namespace clust = minicore::clustering;
using blz::rowMajor;
using blz::rowVector;
using blz::unchecked;
using blz::rowwise;
using minicore::util::sum;
using blz::sum;
using blz::unpadded;
using blz::unaligned;
using blz::padded;
using blz::aligned;
template<typename Matrix, typename CtrT, typename AsnT=blz::DV<uint32_t>, typename CostsT=blz::DV<double>>
py::dict cpp_scluster(const Matrix &mat, int, double beta,
dist::DissimilarityMeasure measure,
std::vector<CtrT> &ctrs,
CostsT &costs,
AsnT &asn,
double temp,
size_t kmeansmaxiter,
Py_ssize_t mbsize,
Py_ssize_t mbn,
void *weights=static_cast<void *>(nullptr),
char wdtype='f')
{
using FT = double;
blz::DV<FT> prior{FT(beta)};
std::tuple<double, double, size_t> clusterret;
blz::DV<FT> rsums = sum<rowwise>(mat);
const auto csums = blz::evaluate(blz::generate(ctrs.size(), [&](auto idx) {return blz::sum(ctrs[idx]);}));
const double psum = beta * mat.columns();
costs = blaze::generate(costs.rows(), costs.columns(), [&](auto r, auto c) {
return cmp::msr_with_prior<float>(measure, row(mat, r, unchecked), ctrs[c], prior, psum, rsums[r], csums[c]);
});
asn = blaze::softmax<blaze::rowwise>(costs);
OMP_PFOR
for(size_t i = 0; i < asn.rows(); ++i) {
auto r = row(asn, i, unchecked);
clust::correct_softmax(row(costs, i, unchecked), r);
}
blz::DV<double> cw;
std::unique_ptr<blz::CustomVector<double, unaligned, unpadded, rowVector>> wview;
if(weights && wdtype > 0) {
if(wdtype != 'd') {
cw.resize(costs.rows());
wview.reset(new blz::CustomVector<double, unaligned, unpadded, rowVector>(cw.data(), cw.size()));
switch(wdtype) {
case 'f': cw = blz::make_cv((float *)weights, costs.rows()); break;
case 'I': case 'i': cw = blz::make_cv((uint32_t *)weights, costs.rows()); break;
case 'L': case 'l': cw = blz::make_cv((uint64_t *)weights, costs.rows()); break;
case 'H': case 'h': cw = blz::make_cv((uint16_t *)weights, costs.rows()); break;
case 'B': case 'b': cw = blz::make_cv((uint8_t *)weights, costs.rows()); break;
default: throw std::invalid_argument("Required: float, double, or uint{8,16,32,64} weights");
}
} else wview.reset(new blz::CustomVector<double, unaligned, unpadded, rowVector>((double *)weights, costs.rows()));
}
// Only one version of perform_soft_clustering compiled (for double weights)
// This takes extra memory/time to copy the weights, but halves or thirds compile-time.
clusterret = minicore::clustering::perform_soft_clustering(mat, measure, prior, ctrs, costs, asn, temp, kmeansmaxiter, mbsize, mbn, wview.get());
auto &[initcost, finalcost, numiter] = clusterret;
auto pyctrs = centers2pylist(ctrs);
//auto pycosts = vec2fnp<decltype(costs), float> (costs);
//auto pyasn = vec2fnp<decltype(asn), uint32_t>(asn);
return py::dict("initcost"_a = initcost, "finalcost"_a = finalcost, "numiter"_a = numiter,
"centers"_a = pyctrs);
}
template<typename Matrix>
py::dict py_scluster(const Matrix &smw,
py::object centers,
dist::DissimilarityMeasure measure,
double beta,
double temp=1.,
size_t kmeansmaxiter=1000,
Py_ssize_t mbsize=-1,
Py_ssize_t mbn=10,
std::string savepref="",
bool use_float=true,
void *weights = (void *)nullptr,
std::string wfmt="f")
{
use_float = true;
assert(beta > 0.);
py::dict retdict;
py::object asns = py::none(), costs = py::none();
std::vector<blz::CompressedVector<float, blz::rowVector>> dvecs;
smw.perform([&](auto &mat) {dvecs = obj2dvec(centers, mat);});
const int k = dvecs.size();
std::vector<Py_ssize_t> shape{Py_ssize_t(smw.rows()), k};
assert(k >= 1);
if(!savepref.empty()) {
std::fprintf(stderr, "Using savepref to mmap cost matrices diretly: %s\n", savepref.data());
std::string cpath = savepref + ".costs." + (use_float ? ".f32": ".f64") + ".npy";
std::string apath = savepref + ".asns." + (use_float ? ".f32": ".f64") + ".npy";
auto mmfn = py::module::import("numpy").attr("memmap");
auto dt = py::dtype(use_float ? "f": "d");
costs = mmfn(py::str(cpath), shape, dt);
asns = mmfn(py::str(apath), shape, dt);
} else {
costs = py::array_t<float>({smw.rows(), smw.columns()}), asns = py::array_t<float>({smw.rows(), smw.columns()});
//else costs = py::array_t<double>({smw.rows(), smw.columns()}), asns = py::array_t<double>({smw.rows(), smw.columns()});
}
void *cp = py::cast<py::array>(costs).request().ptr,
*ap = py::cast<py::array>(asns).request().ptr;
#if 0
if(use_float) {
#endif
blz::CustomMatrix<float, unaligned, unpadded, rowMajor> cm((float *)cp, smw.rows(), k);
blz::CustomMatrix<float, unaligned, unpadded, rowMajor> am((float *)ap, smw.rows(), k);
smw.perform([&](auto &x) {retdict = cpp_scluster(x, k, beta, measure, dvecs, cm, am, temp, kmeansmaxiter, mbsize, mbn, weights, wfmt[0]);});
#if 0
} else {
blz::CustomMatrix<double, unaligned, unpadded, rowMajor> cm((double *)cp, smw.rows(), k);
blz::CustomMatrix<double, unaligned, unpadded, rowMajor> am((double *)ap, smw.rows(), k);
smw.perform([&](auto &x) {retdict = cpp_scluster(x, k, beta, measure, dvecs, cm, am, temp, kmeansmaxiter, mbsize, mbn, weights, wfmt[0]);});
}
#endif
retdict["costs"] = costs;
retdict["asn"] = asns;
return retdict;
}
#endif