FlashGraph-ng
A new frontier in large-scale graph analysis and data mining
 All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Pages
local_vec_store.h
1 #ifndef __LOCAL_VEC_STORE_H__
2 #define __LOCAL_VEC_STORE_H__
3 
4 /*
5  * Copyright 2015 Open Connectome Project (http://openconnecto.me)
6  * Written by Da Zheng (zhengda1936@gmail.com)
7  *
8  * This file is part of FlashMatrix.
9  *
10  * Licensed under the Apache License, Version 2.0 (the "License");
11  * you may not use this file except in compliance with the License.
12  * You may obtain a copy of the License at
13  *
14  * http://www.apache.org/licenses/LICENSE-2.0
15  *
16  * Unless required by applicable law or agreed to in writing, software
17  * distributed under the License is distributed on an "AS IS" BASIS,
18  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19  * See the License for the specific language governing permissions and
20  * limitations under the License.
21  */
22 
23 #include <string.h>
24 
25 #include "mem_vec_store.h"
26 #include "bulk_operate.h"
27 #include "raw_data_array.h"
28 
29 namespace fm
30 {
31 
32 class data_frame;
33 
34 namespace detail
35 {
36 
37 class matrix_store;
38 
39 }
40 
41 class local_vec_store: public detail::mem_vec_store
42 {
43  const off_t orig_global_start;
44  const size_t orig_length;
45  const char *const_data;
46  char *data;
47  off_t global_start;
48  int node_id;
49 protected:
50  void set_data(const char *const_data, char *data) {
51  this->data = data;
52  this->const_data = const_data;
53  }
54 
55 public:
56  local_vec_store(const char *const_data, char *data, off_t _global_start,
57  size_t length, const scalar_type &type,
58  int node_id): detail::mem_vec_store(length, type), orig_global_start(
59  _global_start), orig_length(length) {
60  this->data = data;
61  this->const_data = const_data;
62  this->global_start = _global_start;
63  this->node_id = node_id;
64  }
65  local_vec_store(const char *const_data, char *data, off_t _global_start,
66  size_t length, size_t entry_size, const scalar_type &type,
67  int node_id): detail::mem_vec_store(length, entry_size,
68  type), orig_global_start(_global_start), orig_length(length) {
69  this->data = data;
70  this->const_data = const_data;
71  this->global_start = _global_start;
72  this->node_id = node_id;
73  }
74 
75  virtual bool expose_sub_vec(off_t start, size_t length) {
76  assert(start + length <= orig_length);
77  global_start = orig_global_start + start;
78  return mem_vec_store::resize(length);
79  }
80  virtual void reset_expose() {
81  global_start = orig_global_start;
82  mem_vec_store::resize(orig_length);
83  }
84 
85  int get_node_id() const {
86  return node_id;
87  }
88 
89  off_t get_global_start() const {
90  return global_start;
91  }
92 
93  off_t get_local_start() const {
94  return global_start - orig_global_start;
95  }
96 
97  typedef std::shared_ptr<local_vec_store> ptr;
98  typedef std::shared_ptr<const local_vec_store> const_ptr;
99 
100  virtual const char *get_raw_arr() const {
101  return const_data;
102  }
103  virtual char *get_raw_arr() {
104  return data;
105  }
106  virtual const char *get_sub_arr(off_t start, off_t end) const {
107  assert(start < end && (size_t) end <= get_length());
108  return const_data + start * get_type().get_size();
109  }
110  virtual char *get_sub_arr(off_t start, off_t end) {
111  assert(start < end && (size_t) end <= get_length());
112  return data + start * get_type().get_size();
113  }
114 
115  std::shared_ptr<data_frame> groupby(
116  const gr_apply_operate<local_vec_store> &op, bool with_val) const;
117 
118  virtual void reset_data() {
119  memset(get_raw_arr(), 0, get_length() * get_entry_size());
120  }
121 
122  virtual void set_data(const set_vec_operate &op) {
123  // We always view a vector as a single-column matrix.
124  op.set(get_raw_arr(), get_length(), global_start);
125  }
126 
127  virtual detail::vec_store::ptr sort_with_index();
128  virtual void sort();
129  virtual bool is_sorted() const;
130  virtual std::shared_ptr<detail::matrix_store> conv2mat(
131  size_t nrow, size_t ncol, bool byrow);
132 
133  local_vec_store::ptr get(std::vector<off_t> &idxs) const;
134 
135  const char *get(off_t idx) const {
136  return const_data + idx * get_entry_size();
137  }
138 
139  char *get(off_t idx) {
140  return data + idx * get_entry_size();
141  }
142 
143  virtual local_vec_store::ptr get_portion(off_t loc, size_t size);
144  virtual local_vec_store::const_ptr get_portion(off_t loc,
145  size_t size) const;
146  virtual bool set_portion(std::shared_ptr<const local_vec_store> store,
147  off_t loc);
148 
149  void set_raw(off_t idx, const char *val) {
150  assert(const_data == data);
151  char *dst = data + idx * get_entry_size();
152  memcpy(dst, val, get_entry_size());
153  }
154 
155  template<class T>
156  T get(off_t idx) const {
157  return *(const T *) (const_data + idx * sizeof(T));
158  }
159  template<class T>
160  void set(off_t idx, T val) {
161  assert(const_data == data);
162  *(T *) (data + idx * sizeof(T)) = val;
163  }
164 
165  /*
166  * The following methods aren't needed by this class and its child classes.
167  */
168 
169  virtual size_t get_portion_size() const {
170  assert(0);
171  return 0;
172  }
173  virtual bool append(std::vector<detail::vec_store::const_ptr>::const_iterator vec_it,
174  std::vector<detail::vec_store::const_ptr>::const_iterator vec_end) {
175  assert(0);
176  return false;
177  }
178  virtual bool append(const detail::vec_store &vec) {
179  assert(0);
180  return false;
181  }
182  virtual detail::vec_store::ptr deep_copy() const {
183  assert(0);
184  return detail::vec_store::ptr();
185  }
186 };
187 
188 class local_ref_vec_store: public local_vec_store
189 {
190  char *orig_data;
191 public:
192  local_ref_vec_store(char *data, off_t global_start, size_t length,
193  const scalar_type &type, int node_id): local_vec_store(data,
194  data, global_start, length, type, node_id) {
195  this->orig_data = data;
196  }
197  virtual bool resize(size_t new_length);
198 
199  virtual detail::vec_store::ptr shallow_copy() {
200  return detail::vec_store::ptr(new local_ref_vec_store(*this));
201  }
202  virtual detail::vec_store::const_ptr shallow_copy() const {
203  return detail::vec_store::const_ptr(new local_ref_vec_store(*this));
204  }
205  virtual bool expose_sub_vec(off_t start, size_t length) {
206  char *new_data = orig_data + start * get_type().get_size();
207  set_data(new_data, new_data);
208  return local_vec_store::expose_sub_vec(start, length);
209  }
210  virtual void reset_expose() {
211  set_data(orig_data, orig_data);
212  local_vec_store::reset_expose();
213  }
214 };
215 
216 class local_cref_vec_store: public local_vec_store
217 {
218  const char *orig_data;
219 public:
220  local_cref_vec_store(const char *data, off_t global_start, size_t length,
221  const scalar_type &type, int node_id): local_vec_store(data,
222  NULL, global_start, length, type, node_id) {
223  this->orig_data = data;
224  }
225  virtual bool resize(size_t new_length);
226 
227  virtual detail::vec_store::ptr shallow_copy() {
228  return detail::vec_store::ptr(new local_cref_vec_store(*this));
229  }
230  virtual detail::vec_store::const_ptr shallow_copy() const {
231  return detail::vec_store::const_ptr(new local_cref_vec_store(*this));
232  }
233  virtual bool expose_sub_vec(off_t start, size_t length) {
234  const char *new_data = orig_data + start * get_type().get_size();
235  set_data(new_data, NULL);
236  return local_vec_store::expose_sub_vec(start, length);
237  }
238  virtual void reset_expose() {
239  set_data(orig_data, NULL);
240  local_vec_store::reset_expose();
241  }
242 };
243 
244 class local_buf_vec_store: public local_vec_store
245 {
246  detail::raw_data_array arr;
247 public:
248  local_buf_vec_store(off_t global_start, size_t length,
249  const scalar_type &type, int node_id): local_vec_store(NULL,
250  NULL, global_start, length, type, node_id),
251  arr(length * type.get_size()) {
252  set_data(arr.get_raw(), arr.get_raw());
253  }
254 
255  virtual bool resize(size_t new_length);
256 
257  virtual detail::vec_store::ptr shallow_copy() {
258  return detail::vec_store::ptr(new local_buf_vec_store(*this));
259  }
260  virtual detail::vec_store::const_ptr shallow_copy() const {
261  return detail::vec_store::const_ptr(new local_buf_vec_store(*this));
262  }
263  virtual bool expose_sub_vec(off_t start, size_t length) {
264  char *new_data = arr.get_raw() + start * get_type().get_size();
265  set_data(new_data, new_data);
266  return local_vec_store::expose_sub_vec(start, length);
267  }
268  virtual void reset_expose() {
269  set_data(arr.get_raw(), arr.get_raw());
270  local_vec_store::reset_expose();
271  }
272 };
273 
274 }
275 
276 #endif