Apache GraphAr C++ Library
The C++ Library for Apache GraphAr
chunk_writer.h
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements. See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership. The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License. You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing,
13  * software distributed under the License is distributed on an
14  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15  * KIND, either express or implied. See the License for the
16  * specific language governing permissions and limitations
17  * under the License.
18  */
19 
20 #pragma once
21 
22 #include <memory>
23 #include <string>
24 #include <vector>
25 
26 #include "graphar/fwd.h"
27 #include "graphar/writer_util.h"
28 
29 // forward declaration
30 namespace arrow {
31 class Table;
32 }
33 
34 namespace graphar {
35 
58  public:
69  explicit VertexPropertyWriter(
70  const std::shared_ptr<VertexInfo>& vertex_info, const std::string& prefix,
71  const ValidateLevel& validate_level = ValidateLevel::no_validate);
72 
78  inline void SetValidateLevel(const ValidateLevel& validate_level) {
79  if (validate_level == ValidateLevel::default_validate) {
80  return;
81  }
82  validate_level_ = validate_level;
83  }
84 
90  inline ValidateLevel GetValidateLevel() const { return validate_level_; }
91 
101  const IdType& count,
102  ValidateLevel validate_level = ValidateLevel::default_validate) const;
103 
116  const std::shared_ptr<arrow::Table>& input_table,
117  const std::shared_ptr<PropertyGroup>& property_group, IdType chunk_index,
118  ValidateLevel validate_level = ValidateLevel::default_validate) const;
119 
131  const std::shared_ptr<arrow::Table>& input_table, IdType chunk_index,
132  ValidateLevel validate_level = ValidateLevel::default_validate) const;
133 
146  const std::shared_ptr<arrow::Table>& input_table,
147  const std::shared_ptr<PropertyGroup>& property_group,
148  IdType start_chunk_index,
149  ValidateLevel validate_level = ValidateLevel::default_validate) const;
150 
162  const std::shared_ptr<arrow::Table>& input_table,
163  IdType start_chunk_index,
164  ValidateLevel validate_level = ValidateLevel::default_validate) const;
165 
174  static Result<std::shared_ptr<VertexPropertyWriter>> Make(
175  const std::shared_ptr<VertexInfo>& vertex_info, const std::string& prefix,
176  const ValidateLevel& validate_level = ValidateLevel::no_validate);
177 
186  static Result<std::shared_ptr<VertexPropertyWriter>> Make(
187  const std::shared_ptr<GraphInfo>& graph_info, const std::string& label,
188  const ValidateLevel& validate_level = ValidateLevel::no_validate);
189 
190  private:
198  Status validate(const IdType& count, ValidateLevel validate_level) const;
199 
208  Status validate(const std::shared_ptr<PropertyGroup>& property_group,
209  IdType chunk_index, ValidateLevel validate_level) const;
210 
220  Status validate(const std::shared_ptr<arrow::Table>& input_table,
221  const std::shared_ptr<PropertyGroup>& property_group,
222  IdType chunk_index, ValidateLevel validate_level) const;
223 
224  Result<std::shared_ptr<arrow::Table>> addIndexColumn(
225  const std::shared_ptr<arrow::Table>& table, IdType chunk_index,
226  IdType chunk_size) const;
227 
228  private:
229  std::shared_ptr<VertexInfo> vertex_info_;
230  std::string prefix_;
231  std::shared_ptr<FileSystem> fs_;
232  ValidateLevel validate_level_;
233 };
234 
258  public:
270  explicit EdgeChunkWriter(
271  const std::shared_ptr<EdgeInfo>& edge_info, const std::string& prefix,
272  AdjListType adj_list_type,
273  const ValidateLevel& validate_level = ValidateLevel::no_validate);
274 
280  void SetValidateLevel(const ValidateLevel& validate_level) {
281  if (validate_level == ValidateLevel::default_validate) {
282  return;
283  }
284  validate_level_ = validate_level;
285  }
286 
292  inline ValidateLevel GetValidateLevel() const { return validate_level_; }
293 
304  IdType vertex_chunk_index, const IdType& count,
305  ValidateLevel validate_level = ValidateLevel::default_validate) const;
306 
316  const IdType& count,
317  ValidateLevel validate_level = ValidateLevel::default_validate) const;
318 
329  const std::shared_ptr<arrow::Table>& input_table,
330  IdType vertex_chunk_index,
331  ValidateLevel validate_level = ValidateLevel::default_validate) const;
332 
344  const std::shared_ptr<arrow::Table>& input_table,
345  IdType vertex_chunk_index, IdType chunk_index,
346  ValidateLevel validate_level = ValidateLevel::default_validate) const;
347 
360  const std::shared_ptr<arrow::Table>& input_table,
361  const std::shared_ptr<PropertyGroup>& property_group,
362  IdType vertex_chunk_index, IdType chunk_index,
363  ValidateLevel validate_level = ValidateLevel::default_validate) const;
364 
376  const std::shared_ptr<arrow::Table>& input_table,
377  IdType vertex_chunk_index, IdType chunk_index,
378  ValidateLevel validate_level = ValidateLevel::default_validate) const;
379 
391  const std::shared_ptr<arrow::Table>& input_table,
392  IdType vertex_chunk_index, IdType chunk_index,
393  ValidateLevel validate_level = ValidateLevel::default_validate) const;
394 
407  const std::shared_ptr<arrow::Table>& input_table,
408  IdType vertex_chunk_index, IdType start_chunk_index = 0,
409  ValidateLevel validate_level = ValidateLevel::default_validate) const;
410 
425  const std::shared_ptr<arrow::Table>& input_table,
426  const std::shared_ptr<PropertyGroup>& property_group,
427  IdType vertex_chunk_index, IdType start_chunk_index = 0,
428  ValidateLevel validate_level = ValidateLevel::default_validate) const;
429 
443  const std::shared_ptr<arrow::Table>& input_table,
444  IdType vertex_chunk_index, IdType start_chunk_index = 0,
445  ValidateLevel validate_level = ValidateLevel::default_validate) const;
446 
460  const std::shared_ptr<arrow::Table>& input_table,
461  IdType vertex_chunk_index, IdType start_chunk_index = 0,
462  ValidateLevel validate_level = ValidateLevel::default_validate) const;
463 
477  const std::shared_ptr<arrow::Table>& input_table,
478  IdType vertex_chunk_index, IdType start_chunk_index = 0,
479  ValidateLevel validate_level = ValidateLevel::default_validate) const;
480 
495  const std::shared_ptr<arrow::Table>& input_table,
496  const std::shared_ptr<PropertyGroup>& property_group,
497  IdType vertex_chunk_index, IdType start_chunk_index = 0,
498  ValidateLevel validate_level = ValidateLevel::default_validate) const;
499 
513  const std::shared_ptr<arrow::Table>& input_table,
514  IdType vertex_chunk_index, IdType start_chunk_index = 0,
515  ValidateLevel validate_level = ValidateLevel::default_validate) const;
516 
530  const std::shared_ptr<arrow::Table>& input_table,
531  IdType vertex_chunk_index, IdType start_chunk_index = 0,
532  ValidateLevel validate_level = ValidateLevel::default_validate) const;
533 
543  static Result<std::shared_ptr<EdgeChunkWriter>> Make(
544  const std::shared_ptr<EdgeInfo>& edge_info, const std::string& prefix,
545  AdjListType adj_list_type,
546  const ValidateLevel& validate_level = ValidateLevel::no_validate);
547 
559  static Result<std::shared_ptr<EdgeChunkWriter>> Make(
560  const std::shared_ptr<GraphInfo>& graph_info,
561  const std::string& src_label, const std::string& edge_label,
562  const std::string& dst_label, AdjListType adj_list_type,
563  const ValidateLevel& validate_level = ValidateLevel::no_validate);
564 
565  private:
575  Status validate(IdType count_or_index1, IdType count_or_index2,
576  ValidateLevel validate_level) const;
577 
588  Status validate(const std::shared_ptr<PropertyGroup>& property_group,
589  IdType vertex_chunk_index, IdType chunk_index,
590  ValidateLevel validate_level) const;
591 
601  Status validate(const std::shared_ptr<arrow::Table>& input_table,
602  IdType vertex_chunk_index,
603  ValidateLevel validate_level) const;
604 
615  Status validate(const std::shared_ptr<arrow::Table>& input_table,
616  IdType vertex_chunk_index, IdType chunk_index,
617  ValidateLevel validate_level) const;
618 
630  Status validate(const std::shared_ptr<arrow::Table>& input_table,
631  const std::shared_ptr<PropertyGroup>& property_group,
632  IdType vertex_chunk_index, IdType chunk_index,
633  ValidateLevel validate_level) const;
634 
642  Result<std::shared_ptr<arrow::Table>> getOffsetTable(
643  const std::shared_ptr<arrow::Table>& input_table,
644  const std::string& column_name, IdType vertex_chunk_index) const;
645 
652  static std::string getSortColumnName(AdjListType adj_list_type);
653 
661  static Result<std::shared_ptr<arrow::Table>> sortTable(
662  const std::shared_ptr<arrow::Table>& input_table,
663  const std::string& column_name);
664 
665  private:
666  std::shared_ptr<EdgeInfo> edge_info_;
667  IdType vertex_chunk_size_;
668  IdType chunk_size_;
669  AdjListType adj_list_type_;
670  std::string prefix_;
671  std::shared_ptr<FileSystem> fs_;
672  ValidateLevel validate_level_;
673 };
674 
675 } // namespace graphar
The writer for edge (adj list, offset and property group) chunks.
Definition: chunk_writer.h:257
void SetValidateLevel(const ValidateLevel &validate_level)
Set the validate level.
Definition: chunk_writer.h:280
Status WritePropertyChunk(const std::shared_ptr< arrow::Table > &input_table, const std::shared_ptr< PropertyGroup > &property_group, IdType vertex_chunk_index, IdType chunk_index, ValidateLevel validate_level=ValidateLevel::default_validate) const
Validate and write a single edge property group for an edge chunk.
Status SortAndWriteAdjListTable(const std::shared_ptr< arrow::Table > &input_table, IdType vertex_chunk_index, IdType start_chunk_index=0, ValidateLevel validate_level=ValidateLevel::default_validate) const
Sort the edges, and write the adj list chunks for the edges of a vertex chunk.
Status WriteAdjListTable(const std::shared_ptr< arrow::Table > &input_table, IdType vertex_chunk_index, IdType start_chunk_index=0, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write the adj list chunks for the edges of a vertex chunk.
Status WritePropertyTable(const std::shared_ptr< arrow::Table > &input_table, const std::shared_ptr< PropertyGroup > &property_group, IdType vertex_chunk_index, IdType start_chunk_index=0, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write chunks of a single property group for the edges of a vertex chunk.
EdgeChunkWriter(const std::shared_ptr< EdgeInfo > &edge_info, const std::string &prefix, AdjListType adj_list_type, const ValidateLevel &validate_level=ValidateLevel::no_validate)
Initialize the EdgeChunkWriter.
static Result< std::shared_ptr< EdgeChunkWriter > > Make(const std::shared_ptr< EdgeInfo > &edge_info, const std::string &prefix, AdjListType adj_list_type, const ValidateLevel &validate_level=ValidateLevel::no_validate)
Construct an EdgeChunkWriter from edge info.
Status WriteOffsetChunk(const std::shared_ptr< arrow::Table > &input_table, IdType vertex_chunk_index, ValidateLevel validate_level=ValidateLevel::default_validate) const
Validate and write the offset chunk for a vertex chunk.
Status WriteChunk(const std::shared_ptr< arrow::Table > &input_table, IdType vertex_chunk_index, IdType chunk_index, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write the adj list and all property groups for an edge chunk.
Status SortAndWriteTable(const std::shared_ptr< arrow::Table > &input_table, IdType vertex_chunk_index, IdType start_chunk_index=0, ValidateLevel validate_level=ValidateLevel::default_validate) const
Sort the edges, and write chunks of the adj list and all property groups for the edges of a vertex ch...
Status SortAndWritePropertyTable(const std::shared_ptr< arrow::Table > &input_table, const std::shared_ptr< PropertyGroup > &property_group, IdType vertex_chunk_index, IdType start_chunk_index=0, ValidateLevel validate_level=ValidateLevel::default_validate) const
Sort the edges, and write chunks of a single property group for the edges of a vertex chunk.
Status WriteAdjListChunk(const std::shared_ptr< arrow::Table > &input_table, IdType vertex_chunk_index, IdType chunk_index, ValidateLevel validate_level=ValidateLevel::default_validate) const
Validate and write the adj list chunk for an edge chunk.
Status WriteEdgesNum(IdType vertex_chunk_index, const IdType &count, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write the number of edges into the file.
Status WriteTable(const std::shared_ptr< arrow::Table > &input_table, IdType vertex_chunk_index, IdType start_chunk_index=0, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write chunks of the adj list and all property groups for the edges of a vertex chunk.
ValidateLevel GetValidateLevel() const
Get the validate level.
Definition: chunk_writer.h:292
Status WriteVerticesNum(const IdType &count, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write the number of vertices into the file.
Status outcome object (success or error)
Definition: status.h:123
The writer for vertex property group chunks.
Definition: chunk_writer.h:57
VertexPropertyWriter(const std::shared_ptr< VertexInfo > &vertex_info, const std::string &prefix, const ValidateLevel &validate_level=ValidateLevel::no_validate)
Initialize the VertexPropertyWriter.
ValidateLevel GetValidateLevel() const
Get the validate level.
Definition: chunk_writer.h:90
Status WriteChunk(const std::shared_ptr< arrow::Table > &input_table, const std::shared_ptr< PropertyGroup > &property_group, IdType chunk_index, ValidateLevel validate_level=ValidateLevel::default_validate) const
Validate and write a single property group for a single vertex chunk.
void SetValidateLevel(const ValidateLevel &validate_level)
Set the validate level.
Definition: chunk_writer.h:78
Status WriteVerticesNum(const IdType &count, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write the number of vertices into the file.
Status WriteTable(const std::shared_ptr< arrow::Table > &input_table, const std::shared_ptr< PropertyGroup > &property_group, IdType start_chunk_index, ValidateLevel validate_level=ValidateLevel::default_validate) const
Write a single property group for multiple vertex chunks to corresponding files.
static Result< std::shared_ptr< VertexPropertyWriter > > Make(const std::shared_ptr< VertexInfo > &vertex_info, const std::string &prefix, const ValidateLevel &validate_level=ValidateLevel::no_validate)
Construct a VertexPropertyWriter from vertex info.