FlashGraph-ng
A new frontier in large-scale graph analysis and data mining
 All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Pages
dotp_matrix_store.h
1 #ifndef __NORM2_MATRIX_STORE_H__
2 #define __NORM2_MATRIX_STORE_H__
3 
4 /*
5  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
6  * Written by Da Zheng (zhengda1936@gmail.com)
7  *
8  * This file is part of FlashMatrix.
9  *
10  * Licensed under the Apache License, Version 2.0 (the "License");
11  * you may not use this file except in compliance with the License.
12  * You may obtain a copy of the License at
13  *
14  * http://www.apache.org/licenses/LICENSE-2.0
15  *
16  * Unless required by applicable law or agreed to in writing, software
17  * distributed under the License is distributed on an "AS IS" BASIS,
18  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19  * See the License for the specific language governing permissions and
20  * limitations under the License.
21  */
22 
23 #include "virtual_matrix_store.h"
24 #include "EM_object.h"
25 
26 namespace fm
27 {
28 
29 namespace eigen
30 {
31 
32 class sub_dotp_matrix_store: public detail::virtual_matrix_store, public detail::EM_object
33 {
34  matrix_store::const_ptr orig_store;
35  std::vector<off_t> idxs;
36  std::vector<double> col_dot_prods;
37 
38  matrix_store::const_ptr materialized;
39 public:
40  sub_dotp_matrix_store(matrix_store::const_ptr orig_store,
41  const std::vector<double> &dot_prods,
42  const std::vector<off_t> &idxs): detail::virtual_matrix_store(
43  orig_store->get_num_rows(), idxs.size(),
44  orig_store->is_in_mem(), orig_store->get_type()) {
45  this->orig_store = orig_store;
46  this->col_dot_prods = dot_prods;
47  this->idxs = idxs;
48  assert(dot_prods.size() == idxs.size());
49  assert(orig_store->get_type() == get_scalar_type<double>());
50  }
51 
52  matrix_store::const_ptr get_orig_store() const {
53  return orig_store;
54  }
55 
56  double get_col_dot(off_t idx) const {
57  return col_dot_prods[idx];
58  }
59 
60  std::string get_name() const {
61  return (boost::format("sub_dotp_mat(%1%,%2%)")
62  % orig_store->get_num_rows() % idxs.size()).str();
63  }
64 
65  virtual std::unordered_map<size_t, size_t> get_underlying_mats() const {
66  return orig_store->get_underlying_mats();
67  }
68 
69  matrix_layout_t store_layout() const {
70  return orig_store->store_layout();
71  }
72 
73  virtual matrix_store::const_ptr get_cols(
74  const std::vector<off_t> &idxs) const {
75  std::vector<off_t> orig_idxs(idxs.size());
76  std::vector<double> sub_dot_prods(idxs.size());
77  for (size_t i = 0; i < idxs.size(); i++) {
78  sub_dot_prods[i] = col_dot_prods[idxs[i]];
79  orig_idxs[i] = this->idxs[idxs[i]];
80  }
81  return matrix_store::const_ptr(new sub_dotp_matrix_store(orig_store,
82  sub_dot_prods, orig_idxs));
83  }
84 
85  virtual detail::local_matrix_store::const_ptr get_portion(size_t start_row,
86  size_t start_col, size_t num_rows, size_t num_cols) const {
87  assert(!orig_store->is_wide());
88  assert(orig_store->store_layout() == matrix_layout_t::L_COL);
89  detail::local_matrix_store::const_ptr orig = orig_store->get_portion(
90  start_row, 0, num_rows, orig_store->get_num_cols());
91  const detail::local_col_matrix_store &col_orig
92  = static_cast<const detail::local_col_matrix_store &>(*orig);
93  detail::local_buf_col_matrix_store *store
94  = new detail::local_buf_col_matrix_store(start_row, start_col,
95  num_rows, num_cols, get_type(), orig->get_node_id());
96  for (size_t i = 0; i < idxs.size(); i++)
97  memcpy(store->get_col(i), col_orig.get_col(idxs[i]),
98  store->get_num_rows() * store->get_entry_size());
99  return detail::local_matrix_store::const_ptr(store);
100  }
101 
102  virtual std::pair<size_t, size_t> get_portion_size() const {
103  assert(!orig_store->is_wide());
104  return std::pair<size_t, size_t>(orig_store->get_portion_size().first,
105  idxs.size());
106  }
107 
108  matrix_store::const_ptr transpose() const {
109  return orig_store->get_cols(idxs)->transpose();
110  }
111 
112  virtual detail::async_cres_t get_portion_async(
113  size_t start_row, size_t start_col, size_t num_rows, size_t num_cols,
114  detail::portion_compute::ptr compute) const {
115  if (materialized == NULL)
116  const_cast<sub_dotp_matrix_store *>(this)->materialized
117  = orig_store->get_cols(idxs);
118  return materialized->get_portion_async(start_row, start_col,
119  num_rows, num_cols, compute);
120  }
121 
122  virtual matrix_store::const_ptr materialize(bool in_mem, int num_nodes) const {
123  assert(0);
124  return matrix_store::ptr();
125  }
126 
127  virtual std::vector<safs::io_interface::ptr> create_ios() const {
128  const detail::EM_object *obj = dynamic_cast<const detail::EM_object *>(
129  orig_store.get());
130  assert(obj);
131  return obj->create_ios();
132  }
133 };
134 
135 /*
136  * This matrix store helps maintain the dot products of each column
137  * of the matrix. When we need to fetch a column, this class returns a special
138  * virtual matrix that has the dot product of that column.
139  */
140 class dotp_matrix_store: public detail::virtual_matrix_store, public detail::EM_object
141 {
142  matrix_store::const_ptr orig_store;
143  std::vector<double> col_dot_prods;
144 
145  dotp_matrix_store(matrix_store::const_ptr orig_store): detail::virtual_matrix_store(
146  orig_store->get_num_rows(), orig_store->get_num_cols(),
147  orig_store->is_in_mem(), orig_store->get_type()) {
148  this->orig_store = orig_store;
149 
150  dense_matrix::ptr mat = dense_matrix::create(orig_store);
151  const bulk_uoperate *op = get_type().get_basic_uops().get_op(
152  basic_uops::op_idx::SQ);
153  dense_matrix::ptr sq_mat = mat->sapply(bulk_uoperate::conv2ptr(*op));
154  vector::ptr sums = sq_mat->col_sum();
155 
156  const fm::detail::smp_vec_store &smp_res
157  = dynamic_cast<const fm::detail::smp_vec_store &>(sums->get_data());
158  col_dot_prods.resize(sums->get_length());
159  for (size_t i = 0; i < sums->get_length(); i++)
160  col_dot_prods[i] = smp_res.get<double>(i);
161  }
162 public:
163  typedef std::shared_ptr<dotp_matrix_store> ptr;
164 
165  static ptr create(matrix_store::const_ptr store) {
166  return ptr(new dotp_matrix_store(store));
167  }
168 
169  const std::vector<double> get_col_dot_prods() const {
170  return col_dot_prods;
171  }
172 
173  virtual matrix_store::const_ptr get_cols(
174  const std::vector<off_t> &idxs) const {
175  std::vector<double> sub_dot_prods(idxs.size());
176  for (size_t i = 0; i < idxs.size(); i++)
177  sub_dot_prods[i] = col_dot_prods[idxs[i]];
178  return matrix_store::const_ptr(new sub_dotp_matrix_store(orig_store,
179  sub_dot_prods, idxs));
180  }
181 
182  std::string get_name() const {
183  return orig_store->get_name();
184  }
185 
186  matrix_layout_t store_layout() const {
187  return orig_store->store_layout();
188  }
189 
190  matrix_store::const_ptr transpose() const {
191  return orig_store->transpose();
192  }
193 
194  virtual detail::async_cres_t get_portion_async(
195  size_t start_row, size_t start_col, size_t num_rows, size_t num_cols,
196  detail::portion_compute::ptr compute) const {
197  return orig_store->get_portion_async(start_row, start_col, num_rows,
198  num_cols, compute);
199  }
200  virtual detail::local_matrix_store::const_ptr get_portion(size_t start_row,
201  size_t start_col, size_t num_rows, size_t num_cols) const {
202  return orig_store->get_portion(start_row, start_col, num_rows, num_cols);
203  }
204  virtual std::pair<size_t, size_t> get_portion_size() const {
205  return orig_store->get_portion_size();
206  }
207 
208  virtual matrix_store::const_ptr materialize(bool in_mem, int num_nodes) const {
209  if (orig_store->is_virtual()) {
210  const detail::virtual_matrix_store *store
211  = static_cast<const detail::virtual_matrix_store *>(
212  orig_store.get());
213  return store->materialize(in_mem, num_nodes);
214  }
215  else
216  return matrix_store::ptr(new dotp_matrix_store(*this));
217  }
218 
219  virtual int get_num_nodes() const {
220  return orig_store->get_num_nodes();
221  }
222  virtual std::unordered_map<size_t, size_t> get_underlying_mats() const {
223  return orig_store->get_underlying_mats();
224  }
225 
226  virtual std::vector<safs::io_interface::ptr> create_ios() const {
227  const detail::EM_object *obj = dynamic_cast<const detail::EM_object *>(
228  orig_store.get());
229  assert(obj);
230  return obj->create_ios();
231  }
232 };
233 
234 }
235 
236 }
237 
238 #endif