Skip to content

Commit

Permalink
REVERT: must change the package so that I can go get this for now. Th…
Browse files Browse the repository at this point in the history
…is commit will be reverted
  • Loading branch information
shlomi-dr committed Jan 9, 2022
1 parent 5f74d27 commit a60aec2
Show file tree
Hide file tree
Showing 64 changed files with 405 additions and 405 deletions.
22 changes: 11 additions & 11 deletions common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ import (

"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/xitongsys/parquet-go/parquet"
"github.com/xitongsys/parquet-go/types"
"github.com/shlomi-dr/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/types"
)

// `parquet:"name=Name, type=FIXED_LEN_BYTE_ARRAY, length=12"`
Expand Down Expand Up @@ -531,7 +531,7 @@ func DeepCopy(src, dst interface{}) {
return
}

//Get key tag map for map
// Get key tag map for map
func GetKeyTagMap(src *Tag) *Tag {
res := NewTag()
res.InName = "Key"
Expand All @@ -549,7 +549,7 @@ func GetKeyTagMap(src *Tag) *Tag {
return res
}

//Get value tag map for map
// Get value tag map for map
func GetValueTagMap(src *Tag) *Tag {
res := NewTag()
res.InName = "Value"
Expand All @@ -567,7 +567,7 @@ func GetValueTagMap(src *Tag) *Tag {
return res
}

//Convert string to a golang variable name
// Convert string to a golang variable name
func StringToVariableName(str string) string {
ln := len(str)
if ln <= 0 {
Expand All @@ -589,7 +589,7 @@ func StringToVariableName(str string) string {
return name
}

//Convert the first letter of a string to uppercase
// Convert the first letter of a string to uppercase
func HeadToUpper(str string) string {
ln := len(str)
if ln <= 0 {
Expand All @@ -600,7 +600,7 @@ func HeadToUpper(str string) string {
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') {
return strings.ToUpper(str[0:1]) + str[1:]
}
//handle non-alpha prefix such as "_"
// handle non-alpha prefix such as "_"
return "PARGO_PREFIX_" + str
}

Expand Down Expand Up @@ -929,7 +929,7 @@ func (table decimalStringFuncTable) MinMaxSize(minVal interface{}, maxVal interf
return Min(table, minVal, val), Max(table, maxVal, val), int32(len(val.(string)))
}

//Get the size of a parquet value
// Get the size of a parquet value
func SizeOf(val reflect.Value) int64 {
var size int64
switch val.Type().Kind() {
Expand Down Expand Up @@ -978,17 +978,17 @@ func ReformPathStr(pathStr string) string {
return strings.ReplaceAll(pathStr, ".", "\x01")
}

//Convert path slice to string
// Convert path slice to string
func PathToStr(path []string) string {
return strings.Join(path, PAR_GO_PATH_DELIMITER)
}

//Convert string to path slice
// Convert string to path slice
func StrToPath(str string) []string {
return strings.Split(str, PAR_GO_PATH_DELIMITER)
}

//Get the pathStr index in a path
// Get the pathStr index in a path
func PathStrIndex(str string) int {
return len(strings.Split(str, PAR_GO_PATH_DELIMITER))
}
Expand Down
4 changes: 2 additions & 2 deletions common/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (
"reflect"
"testing"

"github.com/xitongsys/parquet-go/parquet"
. "github.com/xitongsys/parquet-go/types"
"github.com/shlomi-dr/parquet-go/parquet"
. "github.com/shlomi-dr/parquet-go/types"
)

func TestHeadToUpper(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion compress/compress.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package compress

import (
"fmt"
"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

type Compressor struct {
Expand Down
2 changes: 1 addition & 1 deletion compress/default.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package compress

import "github.com/xitongsys/parquet-go/parquet"
import "github.com/shlomi-dr/parquet-go/parquet"

func init() {
compressors[parquet.CompressionCodec_UNCOMPRESSED] = &Compressor{
Expand Down
2 changes: 1 addition & 1 deletion compress/gzip.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package compress
import (
"bytes"
"github.com/klauspost/compress/gzip"
"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
"io/ioutil"
"sync"
)
Expand Down
2 changes: 1 addition & 1 deletion compress/gzip_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package compress

import (
"bytes"
"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
"testing"
)

Expand Down
2 changes: 1 addition & 1 deletion compress/lz4.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"sync"

"github.com/pierrec/lz4/v4"
"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

func init() {
Expand Down
2 changes: 1 addition & 1 deletion compress/snappy.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package compress

import (
"github.com/golang/snappy"
"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

func init() {
Expand Down
2 changes: 1 addition & 1 deletion compress/zstd.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ package compress

import (
"github.com/klauspost/compress/zstd"
"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

func init() {
Expand Down
10 changes: 5 additions & 5 deletions encoding/encodingread.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"io"
"math"

"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

func ReadPlain(bytesReader *bytes.Reader, dataType parquet.Type, cnt uint64, bitWidth uint64) ([]interface{}, error) {
Expand Down Expand Up @@ -142,7 +142,7 @@ func ReadUnsignedVarInt(bytesReader *bytes.Reader) (uint64, error) {
return res, err
}

//RLE return res is []INT64
// RLE return res is []INT64
func ReadRLE(bytesReader *bytes.Reader, header uint64, bitWidth uint64) ([]interface{}, error) {
var err error
var res []interface{}
Expand All @@ -166,7 +166,7 @@ func ReadRLE(bytesReader *bytes.Reader, header uint64, bitWidth uint64) ([]inter
return res, err
}

//return res is []INT64
// return res is []INT64
func ReadBitPacked(bytesReader *bytes.Reader, header uint64, bitWidth uint64) ([]interface{}, error) {
var err error
numGroup := (header >> 1)
Expand Down Expand Up @@ -227,7 +227,7 @@ func ReadBitPacked(bytesReader *bytes.Reader, header uint64, bitWidth uint64) ([
return res, err
}

//res is INT64
// res is INT64
func ReadRLEBitPackedHybrid(bytesReader *bytes.Reader, bitWidth uint64, length uint64) ([]interface{}, error) {
res := make([]interface{}, 0)
if length <= 0 {
Expand Down Expand Up @@ -325,7 +325,7 @@ func ReadDeltaBinaryPackedINT32(bytesReader *bytes.Reader) ([]interface{}, error
return res[:numValues], err
}

//res is INT64
// res is INT64
func ReadDeltaBinaryPackedINT64(bytesReader *bytes.Reader) ([]interface{}, error) {
var (
err error
Expand Down
2 changes: 1 addition & 1 deletion encoding/encodingread_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"testing"
"unsafe"

"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

func TestReadPlainBOOLEAN(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions encoding/encodingwrite.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import (
"math/bits"
"reflect"

"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

func ToInt64(nums []interface{}) []int64 { //convert bool/int values to int64 values
func ToInt64(nums []interface{}) []int64 { // convert bool/int values to int64 values
ln := len(nums)
res := make([]int64, ln)
if ln <= 0 {
Expand Down
2 changes: 1 addition & 1 deletion encoding/encodingwrite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"math/bits"
"testing"

"github.com/xitongsys/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/parquet"
)

func TestToInt64(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions example/arrow_to_parquet.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ import (
"github.com/apache/arrow/go/arrow"
"github.com/apache/arrow/go/arrow/array"
"github.com/apache/arrow/go/arrow/memory"
"github.com/shlomi-dr/parquet-go/reader"
"github.com/shlomi-dr/parquet-go/writer"
"github.com/xitongsys/parquet-go-source/local"
"github.com/xitongsys/parquet-go/reader"
"github.com/xitongsys/parquet-go/writer"
)

func main() {
Expand Down
14 changes: 7 additions & 7 deletions example/column_read.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import (
"log"
"time"

"github.com/shlomi-dr/parquet-go/common"
"github.com/shlomi-dr/parquet-go/reader"
"github.com/shlomi-dr/parquet-go/writer"
"github.com/xitongsys/parquet-go-source/local"
"github.com/xitongsys/parquet-go/common"
"github.com/xitongsys/parquet-go/reader"
"github.com/xitongsys/parquet-go/writer"
)

type Student struct {
Expand All @@ -23,7 +23,7 @@ type Student struct {

func main() {
var err error
//write
// write
fw, err := local.NewLocalFileWriter("column.parquet")
if err != nil {
log.Println("Can't create file", err)
Expand Down Expand Up @@ -60,7 +60,7 @@ func main() {
var names, classes, scores_key, scores_value, ids []interface{}
var rls, dls []int32

///read
// /read
fr, err := local.NewLocalFileReader("column.parquet")
if err != nil {
log.Println("Can't open file", err)
Expand All @@ -73,7 +73,7 @@ func main() {
}
num = int64(pr.GetNumRows())

pr.SkipRowsByPath(common.ReformPathStr("parquet_go_root.name"), 5) //skip the first five rows
pr.SkipRowsByPath(common.ReformPathStr("parquet_go_root.name"), 5) // skip the first five rows
names, rls, dls, err = pr.ReadColumnByPath(common.ReformPathStr("parquet_go_root.name"), num)
log.Println("name", names, rls, dls, err)

Expand All @@ -85,7 +85,7 @@ func main() {
log.Println("parquet_go_root.scores_key", scores_key, err)
log.Println("parquet_go_root.scores_value", scores_value, err)

pr.SkipRowsByIndex(2, 5) //skip the first five rows
pr.SkipRowsByIndex(2, 5) // skip the first five rows
ids, _, _, _ = pr.ReadColumnByIndex(2, num)
log.Println(ids)

Expand Down
14 changes: 7 additions & 7 deletions example/convert_to_json.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ import (
"log"
"time"

"github.com/shlomi-dr/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/reader"
"github.com/shlomi-dr/parquet-go/writer"
"github.com/xitongsys/parquet-go-source/local"
"github.com/xitongsys/parquet-go/parquet"
"github.com/xitongsys/parquet-go/reader"
"github.com/xitongsys/parquet-go/writer"
)

type Student struct {
Expand All @@ -19,7 +19,7 @@ type Student struct {
Sex bool `parquet:"name=sex, type=BOOLEAN"`
Day int32 `parquet:"name=day, type=INT32, convertedtype=DATE"`
Scores map[string]int32 `parquet:"name=scores, type=MAP, keytype=BYTE_ARRAY, keyconvertedtype=UTF8, valuetype=INT32"`
Ignored int32 //without parquet tag and won't write
Ignored int32 // without parquet tag and won't write
}

func main() {
Expand All @@ -30,14 +30,14 @@ func main() {
return
}

//write
// write
pw, err := writer.NewParquetWriter(fw, new(Student), 4)
if err != nil {
log.Println("Can't create parquet writer", err)
return
}

pw.RowGroupSize = 128 * 1024 * 1024 //128M
pw.RowGroupSize = 128 * 1024 * 1024 // 128M
pw.CompressionType = parquet.CompressionCodec_SNAPPY
num := 10
for i := 0; i < num; i++ {
Expand Down Expand Up @@ -65,7 +65,7 @@ func main() {
log.Println("Write Finished")
fw.Close()

///read
// /read
fr, err := local.NewLocalFileReader("to_json.parquet")
if err != nil {
log.Println("Can't open file")
Expand Down
6 changes: 3 additions & 3 deletions example/csv_to_parquet.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ import (
"log"
"os"

"github.com/shlomi-dr/parquet-go/parquet"
"github.com/shlomi-dr/parquet-go/writer"
"github.com/xitongsys/parquet-go-source/local"
"github.com/xitongsys/parquet-go/parquet"
"github.com/xitongsys/parquet-go/writer"
)

type Shoe struct {
Expand All @@ -32,7 +32,7 @@ func main() {
return
}

pw.RowGroupSize = 128 * 1024 * 1024 //128M
pw.RowGroupSize = 128 * 1024 * 1024 // 128M
pw.CompressionType = parquet.CompressionCodec_SNAPPY

csvFile, _ := os.Open("data/shoes.csv")
Expand Down
4 changes: 2 additions & 2 deletions example/csv_write.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"fmt"
"log"

"github.com/shlomi-dr/parquet-go/writer"
"github.com/xitongsys/parquet-go-source/local"
"github.com/xitongsys/parquet-go/writer"
)

func main() {
Expand All @@ -18,7 +18,7 @@ func main() {
"name=Sex, type=BOOLEAN",
}

//write
// write
fw, err := local.NewLocalFileWriter("csv.parquet")
if err != nil {
log.Println("Can't open file", err)
Expand Down
Loading

0 comments on commit a60aec2

Please sign in to comment.