From 108b9f12c4c521c501db2d33c28a599d4720c255 Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 01/11] Added : improving loadup speed of EDS by a significant margin --- pkg/od/parser.go | 191 ++++++++++++++++++++++++++++++++++++++++++++ pkg/od/variable.go | 193 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 384 insertions(+) diff --git a/pkg/od/parser.go b/pkg/od/parser.go index 8175715..136ffec 100644 --- a/pkg/od/parser.go +++ b/pkg/od/parser.go @@ -2,10 +2,12 @@ package od import ( "archive/zip" + "bufio" "bytes" "embed" "fmt" "io" + "os" "regexp" "strconv" @@ -17,6 +19,10 @@ import ( var f embed.FS var rawDefaultOd []byte +// Get index & subindex matching +var matchIdxRegExp = regexp.MustCompile(`^[0-9A-Fa-f]{4}$`) +var matchSubidxRegExp = regexp.MustCompile(`^([0-9A-Fa-f]{4})sub([0-9A-Fa-f]+)$`) + // Return embeded default object dictionary func Default() *ObjectDictionary { defaultOd, err := Parse(rawDefaultOd, 0) @@ -26,6 +32,191 @@ func Default() *ObjectDictionary { return defaultOd } +// trimSpaces trims spaces efficiently without new allocations +func trimSpaces(b []byte) []byte { + start, end := 0, len(b) + + // Trim left space + for start < end && (b[start] == ' ' || b[start] == '\t') { + start++ + } + // Trim right space + for end > start && (b[end-1] == ' ' || b[end-1] == '\t') { + end-- + } + return b[start:end] +} + +// Parse an EDS file +// file can be either a path or an *os.File or []byte +// Other file types could be supported in the future +func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { + + filename := file.(string) + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + var section string + + od := NewOD() + + entry := &Entry{} + vList := &VariableList{} + var subindex uint8 + + isEntry := false + isSubEntry := false + + var defaultValue string + var parameterName string + var objectType string + var pdoMapping string + var lowLimit string + var highLimit string + var subNumber string + var accessType string + var dataType string + + // Scan all .ini lines + for scanner.Scan() { + + line := trimSpaces(scanner.Bytes()) // Read as []byte to reduce allocations + + // Skip empty lines and comments + if len(line) == 0 || line[0] == ';' || line[0] == '#' { + continue + } + + // Handle section headers: [section] + if line[0] == '[' && line[len(line)-1] == ']' { + + // New section, this means we have finished building + // Previous one, so take all the values and update the section + if parameterName != "" && isEntry { + entry.Name = parameterName + vList, err = PopulateEntry( + entry, + nodeId, + parameterName, + defaultValue, + objectType, + pdoMapping, + lowLimit, + highLimit, + accessType, + dataType, + subNumber, + ) + if err != nil { + return nil, fmt.Errorf("failed to create new entry %v", err) + } + } else if parameterName != "" && isSubEntry { + err = PopulateSubEntry( + entry, + vList, + nodeId, + parameterName, + defaultValue, + objectType, + pdoMapping, + lowLimit, + highLimit, + accessType, + dataType, + subindex, + ) + if err != nil { + return nil, fmt.Errorf("failed to create sub entry %v", err) + } + } + + // Match indexes and not sub indexes + section = string(line[1 : len(line)-1]) + + isEntry = false + isSubEntry = false + + if matchIdxRegExp.MatchString(section) { + + // Add a new entry inside object dictionary + idx, err := strconv.ParseUint(section, 16, 16) + if err != nil { + return nil, err + } + isEntry = true + entry = &Entry{} + entry.Index = uint16(idx) + entry.subEntriesNameMap = map[string]uint8{} + entry.logger = od.logger.With("index", idx) + od.entriesByIndexValue[uint16(idx)] = entry + + } else if matchSubidxRegExp.MatchString(section) { + // Do we need to do smthg ? + // TODO we could get entry to double check if ever something is out of order + isSubEntry = true + // Subindex part is from the 7th letter onwards + sidx, err := strconv.ParseUint(section[7:], 16, 8) + if err != nil { + return nil, err + } + subindex = uint8(sidx) + } + + // Reset all values + defaultValue = "" + parameterName = "" + objectType = "" + pdoMapping = "" + lowLimit = "" + highLimit = "" + subNumber = "" + accessType = "" + dataType = "" + + continue + } + + // We are in a section so we need to populate the given entry + // Parse key-value pairs: key = value + // We will create variables for storing intermediate values + // Once we are at the end of the section + + if equalsIdx := bytes.IndexByte(line, '='); equalsIdx != -1 { + key := string(trimSpaces(line[:equalsIdx])) + value := string(trimSpaces(line[equalsIdx+1:])) + + // We will get the different elements of the entry + switch key { + case "ParameterName": + parameterName = value + case "ObjectType": + objectType = value + case "SubNumber": + subNumber = value + case "AccessType": + accessType = value + case "DataType": + dataType = value + case "LowLimit": + lowLimit = value + case "HighLimit": + highLimit = value + case "DefaultValue": + defaultValue = value + case "PDOMapping": + pdoMapping = value + + } + } + } + + return od, nil +} + // Parse an EDS file // file can be either a path or an *os.File or []byte // Other file types could be supported in the future diff --git a/pkg/od/variable.go b/pkg/od/variable.go index 312872b..25bf73f 100644 --- a/pkg/od/variable.go +++ b/pkg/od/variable.go @@ -21,6 +21,199 @@ func (variable *Variable) DefaultValue() []byte { return variable.valueDefault } +// func PopulateVariable( +// variable *Variable, +// nodeId uint8, +// parameterName string, +// defaultValue string, +// objectType string, +// pdoMapping string, +// lowLimit string, +// highLimit string, +// accessType string, +// dataType string, +// subNumber string, +// ) error { + +// if dataType == "" { +// return fmt.Errorf("need data type") +// } +// dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) +// if err != nil { +// return fmt.Errorf("failed to parse object type %v", err) +// } + +// // Get Attribute +// dType := uint8(dataTypeUint) +// attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) + +// variable.Name = parameterName +// variable.DataType = dType +// variable.Attribute = attribute +// variable.SubIndex = 0 + +// if strings.Contains(defaultValue, "$NODEID") { +// re := regexp.MustCompile(`\+?\$NODEID\+?`) +// defaultValue = re.ReplaceAllString(defaultValue, "") +// } else { +// nodeId = 0 +// } +// variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) +// if err != nil { +// return fmt.Errorf("failed to parse 'DefaultValue' for x%x|x%x, because %v (datatype :x%x)", "", 0, err, variable.DataType) +// } +// variable.value = make([]byte, len(variable.valueDefault)) +// copy(variable.value, variable.valueDefault) +// return nil +// } + +func PopulateEntry( + entry *Entry, + nodeId uint8, + parameterName string, + defaultValue string, + objectType string, + pdoMapping string, + lowLimit string, + highLimit string, + accessType string, + dataType string, + subNumber string, +) (*VariableList, error) { + + oType := uint8(0) + // Determine object type + // If no object type, default to 7 (CiA spec) + if objectType == "" { + oType = 7 + } else { + oTypeUint, err := strconv.ParseUint(objectType, 0, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse object type %v", err) + } + oType = uint8(oTypeUint) + } + entry.ObjectType = oType + + // Add necessary stuff depending on oType + switch oType { + + case ObjectTypeVAR, ObjectTypeDOMAIN: + variable := &Variable{} + if dataType == "" { + return nil, fmt.Errorf("need data type") + } + dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse object type %v", err) + } + + // Get Attribute + dType := uint8(dataTypeUint) + attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) + + variable.Name = parameterName + variable.DataType = dType + variable.Attribute = attribute + variable.SubIndex = 0 + + if strings.Contains(defaultValue, "$NODEID") { + re := regexp.MustCompile(`\+?\$NODEID\+?`) + defaultValue = re.ReplaceAllString(defaultValue, "") + } else { + nodeId = 0 + } + variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) + if err != nil { + return nil, fmt.Errorf("failed to parse 'DefaultValue' for x%x|x%x, because %v (datatype :x%x)", "", 0, err, variable.DataType) + } + variable.value = make([]byte, len(variable.valueDefault)) + copy(variable.value, variable.valueDefault) + entry.object = variable + return nil, nil + + case ObjectTypeARRAY: + // Array objects do not allow holes in subindex numbers + // So pre-init slice up to subnumber + sub, err := strconv.ParseUint(subNumber, 0, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse subnumber %v", err) + } + vList := NewArray(uint8(sub)) + entry.object = vList + return vList, nil + + case ObjectTypeRECORD: + // Record objects allow holes in mapping + // Sub-objects will be added with "append" + vList := NewRecord() + entry.object = vList + return vList, nil + + default: + return nil, fmt.Errorf("unknown object type %v", oType) + } +} + +func PopulateSubEntry( + entry *Entry, + vlist *VariableList, + nodeId uint8, + parameterName string, + defaultValue string, + objectType string, + pdoMapping string, + lowLimit string, + highLimit string, + accessType string, + dataType string, + subIndex uint8, +) error { + if dataType == "" { + return fmt.Errorf("need data type") + } + dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) + if err != nil { + return fmt.Errorf("failed to parse object type %v", err) + } + + // Get Attribute + dType := uint8(dataTypeUint) + attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) + + variable := &Variable{ + Name: parameterName, + DataType: byte(dataTypeUint), + Attribute: attribute, + SubIndex: 0, + } + if strings.Contains(defaultValue, "$NODEID") { + re := regexp.MustCompile(`\+?\$NODEID\+?`) + defaultValue = re.ReplaceAllString(defaultValue, "") + } else { + nodeId = 0 + } + variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) + if err != nil { + return fmt.Errorf("failed to parse 'DefaultValue' %v %v %v", err, defaultValue, variable.DataType) + } + variable.value = make([]byte, len(variable.valueDefault)) + copy(variable.value, variable.valueDefault) + + switch entry.ObjectType { + case ObjectTypeARRAY: + vlist.Variables[subIndex] = variable + entry.subEntriesNameMap[parameterName] = subIndex + case ObjectTypeRECORD: + vlist.Variables = append(vlist.Variables, variable) + entry.subEntriesNameMap[parameterName] = subIndex + default: + return fmt.Errorf("add member not supported for ObjectType : %v", entry.ObjectType) + } + + return nil +} + // Create variable from section entry func NewVariableFromSection( section *ini.Section, From 0bdd0e662e592e095be8a861f21ba398061a4006 Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 02/11] Changed : create a seperate file for v2 parser. Changed : other small improvements : use io.Copy, scan lines on our own, reduce regexp lookups by elimitating index search for subindexes Added : record with predefined size to reduce append(..) calls --- pkg/od/od.go | 4 + pkg/od/parser.go | 187 --------------------- pkg/od/parser_v2.go | 400 ++++++++++++++++++++++++++++++++++++++++++++ pkg/od/variable.go | 193 --------------------- 4 files changed, 404 insertions(+), 380 deletions(-) create mode 100644 pkg/od/parser_v2.go diff --git a/pkg/od/od.go b/pkg/od/od.go index 34b0f14..30430a7 100644 --- a/pkg/od/od.go +++ b/pkg/od/od.go @@ -286,6 +286,10 @@ func NewRecord() *VariableList { return &VariableList{objectType: ObjectTypeRECORD, Variables: make([]*Variable, 0)} } +func NewRecordWithLength(length uint8) *VariableList { + return &VariableList{objectType: ObjectTypeRECORD, Variables: make([]*Variable, length)} +} + func NewArray(length uint8) *VariableList { return &VariableList{objectType: ObjectTypeARRAY, Variables: make([]*Variable, length)} } diff --git a/pkg/od/parser.go b/pkg/od/parser.go index 136ffec..ffc5c2b 100644 --- a/pkg/od/parser.go +++ b/pkg/od/parser.go @@ -2,12 +2,10 @@ package od import ( "archive/zip" - "bufio" "bytes" "embed" "fmt" "io" - "os" "regexp" "strconv" @@ -32,191 +30,6 @@ func Default() *ObjectDictionary { return defaultOd } -// trimSpaces trims spaces efficiently without new allocations -func trimSpaces(b []byte) []byte { - start, end := 0, len(b) - - // Trim left space - for start < end && (b[start] == ' ' || b[start] == '\t') { - start++ - } - // Trim right space - for end > start && (b[end-1] == ' ' || b[end-1] == '\t') { - end-- - } - return b[start:end] -} - -// Parse an EDS file -// file can be either a path or an *os.File or []byte -// Other file types could be supported in the future -func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { - - filename := file.(string) - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - var section string - - od := NewOD() - - entry := &Entry{} - vList := &VariableList{} - var subindex uint8 - - isEntry := false - isSubEntry := false - - var defaultValue string - var parameterName string - var objectType string - var pdoMapping string - var lowLimit string - var highLimit string - var subNumber string - var accessType string - var dataType string - - // Scan all .ini lines - for scanner.Scan() { - - line := trimSpaces(scanner.Bytes()) // Read as []byte to reduce allocations - - // Skip empty lines and comments - if len(line) == 0 || line[0] == ';' || line[0] == '#' { - continue - } - - // Handle section headers: [section] - if line[0] == '[' && line[len(line)-1] == ']' { - - // New section, this means we have finished building - // Previous one, so take all the values and update the section - if parameterName != "" && isEntry { - entry.Name = parameterName - vList, err = PopulateEntry( - entry, - nodeId, - parameterName, - defaultValue, - objectType, - pdoMapping, - lowLimit, - highLimit, - accessType, - dataType, - subNumber, - ) - if err != nil { - return nil, fmt.Errorf("failed to create new entry %v", err) - } - } else if parameterName != "" && isSubEntry { - err = PopulateSubEntry( - entry, - vList, - nodeId, - parameterName, - defaultValue, - objectType, - pdoMapping, - lowLimit, - highLimit, - accessType, - dataType, - subindex, - ) - if err != nil { - return nil, fmt.Errorf("failed to create sub entry %v", err) - } - } - - // Match indexes and not sub indexes - section = string(line[1 : len(line)-1]) - - isEntry = false - isSubEntry = false - - if matchIdxRegExp.MatchString(section) { - - // Add a new entry inside object dictionary - idx, err := strconv.ParseUint(section, 16, 16) - if err != nil { - return nil, err - } - isEntry = true - entry = &Entry{} - entry.Index = uint16(idx) - entry.subEntriesNameMap = map[string]uint8{} - entry.logger = od.logger.With("index", idx) - od.entriesByIndexValue[uint16(idx)] = entry - - } else if matchSubidxRegExp.MatchString(section) { - // Do we need to do smthg ? - // TODO we could get entry to double check if ever something is out of order - isSubEntry = true - // Subindex part is from the 7th letter onwards - sidx, err := strconv.ParseUint(section[7:], 16, 8) - if err != nil { - return nil, err - } - subindex = uint8(sidx) - } - - // Reset all values - defaultValue = "" - parameterName = "" - objectType = "" - pdoMapping = "" - lowLimit = "" - highLimit = "" - subNumber = "" - accessType = "" - dataType = "" - - continue - } - - // We are in a section so we need to populate the given entry - // Parse key-value pairs: key = value - // We will create variables for storing intermediate values - // Once we are at the end of the section - - if equalsIdx := bytes.IndexByte(line, '='); equalsIdx != -1 { - key := string(trimSpaces(line[:equalsIdx])) - value := string(trimSpaces(line[equalsIdx+1:])) - - // We will get the different elements of the entry - switch key { - case "ParameterName": - parameterName = value - case "ObjectType": - objectType = value - case "SubNumber": - subNumber = value - case "AccessType": - accessType = value - case "DataType": - dataType = value - case "LowLimit": - lowLimit = value - case "HighLimit": - highLimit = value - case "DefaultValue": - defaultValue = value - case "PDOMapping": - pdoMapping = value - - } - } - } - - return od, nil -} - // Parse an EDS file // file can be either a path or an *os.File or []byte // Other file types could be supported in the future diff --git a/pkg/od/parser_v2.go b/pkg/od/parser_v2.go new file mode 100644 index 0000000..c7707a8 --- /dev/null +++ b/pkg/od/parser_v2.go @@ -0,0 +1,400 @@ +package od + +import ( + "bytes" + "fmt" + "io" + "os" + "regexp" + "strconv" + "strings" +) + +// v2 of OD parser, this implementation is ~10x faster +// than the previous one but has some caveats : +// +// - it expects OD definitions to be "in order" i.e. +// for example this is not possible : +// [1000] +// ... +// [1000sub0] +// ... +// [1001sub0] +// ... +// [1000sub1] +// ... +// [1001] +// +// The remaining bottlenecks are the following : +// +// - regexp are pretty slow, not sure if would could do much better +// - bytes to string conversions for values create a lot of unnecessary allocation. +// As values are mostly stored in bytes anyway, we could remove this step. +// - file i/o ==> not much to do here +func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { + + filename := file.(string) + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + bu := &bytes.Buffer{} + io.Copy(bu, f) + buffer := bu.Bytes() + + od := NewOD() + + var section string + entry := &Entry{} + vList := &VariableList{} + start := 0 + isEntry := false + isSubEntry := false + subindex := uint8(0) + + var defaultValue string + var parameterName string + var objectType string + var pdoMapping string + var lowLimit string + var highLimit string + var subNumber string + var accessType string + var dataType string + + // Scan hole EDS file + for i, b := range buffer { + + if b != '\n' { + continue + } + + // New line detected + lineRaw := buffer[start:i] + start = i + 1 + + // Skip if less than 2 chars + if len(lineRaw) < 2 { + continue + } + + line := trimSpaces(lineRaw) + + // Skip empty lines and comments + if len(line) == 0 || line[0] == ';' || line[0] == '#' { + continue + } + + // Handle section headers: [section] + if line[0] == '[' && line[len(line)-2] == ']' { + + // A section should be of length 4 at least + if len(line) < 4 { + continue + } + + // New section, this means we have finished building + // Previous one, so take all the values and update the section + if parameterName != "" && isEntry { + entry.Name = parameterName + od.entriesByIndexName[parameterName] = entry + vList, err = populateEntry( + entry, + nodeId, + parameterName, + defaultValue, + objectType, + pdoMapping, + lowLimit, + highLimit, + accessType, + dataType, + subNumber, + ) + + if err != nil { + return nil, fmt.Errorf("failed to create new entry %v", err) + } + + } else if parameterName != "" && isSubEntry { + err = populateSubEntry( + entry, + vList, + nodeId, + parameterName, + defaultValue, + objectType, + pdoMapping, + lowLimit, + highLimit, + accessType, + dataType, + subindex, + ) + + if err != nil { + return nil, fmt.Errorf("failed to create sub entry %v", err) + } + } + + isEntry = false + isSubEntry = false + sectionBytes := line[1 : len(line)-2] + + // Check if a sub entry or the actual entry + // A subentry should be more than 4 bytes long + subSection := sectionBytes[4:] + if len(subSection) < 4 && matchIdxRegExp.Match(sectionBytes) { + section = string(sectionBytes) + + // Add a new entry inside object dictionary + idx, err := strconv.ParseUint(section, 16, 16) + if err != nil { + return nil, err + } + isEntry = true + entry = &Entry{} + entry.Index = uint16(idx) + entry.subEntriesNameMap = map[string]uint8{} + entry.logger = od.logger + od.entriesByIndexValue[uint16(idx)] = entry + + } else if matchSubidxRegExp.Match(sectionBytes) { + section = string(sectionBytes) + // TODO we could get entry to double check if ever something is out of order + isSubEntry = true + // Subindex part is from the 7th letter onwards + sidx, err := strconv.ParseUint(section[7:], 16, 8) + if err != nil { + return nil, err + } + subindex = uint8(sidx) + } + + // Reset all values + defaultValue = "" + parameterName = "" + objectType = "" + pdoMapping = "" + lowLimit = "" + highLimit = "" + subNumber = "" + accessType = "" + dataType = "" + + continue + } + + // We are in a section so we need to populate the given entry + // Parse key-value pairs: key = value + // We will create variables for storing intermediate values + // Once we are at the end of the section + + if equalsIdx := bytes.IndexByte(line, '='); equalsIdx != -1 { + key := string(trimSpaces(line[:equalsIdx])) + value := string(trimSpacesAndr(line[equalsIdx+1:])) + + // We will get the different elements of the entry + switch key { + case "ParameterName": + parameterName = value + case "ObjectType": + objectType = value + case "SubNumber": + subNumber = value + case "AccessType": + accessType = value + case "DataType": + dataType = value + case "LowLimit": + lowLimit = value + case "HighLimit": + highLimit = value + case "DefaultValue": + defaultValue = value + case "PDOMapping": + pdoMapping = value + + } + } + } + return od, nil +} + +func populateEntry( + entry *Entry, + nodeId uint8, + parameterName string, + defaultValue string, + objectType string, + pdoMapping string, + lowLimit string, + highLimit string, + accessType string, + dataType string, + subNumber string, +) (*VariableList, error) { + + oType := uint8(0) + // Determine object type + // If no object type, default to 7 (CiA spec) + if objectType == "" { + oType = 7 + } else { + oTypeUint, err := strconv.ParseUint(objectType, 0, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse object type %v", err) + } + oType = uint8(oTypeUint) + } + entry.ObjectType = oType + + // Add necessary stuff depending on oType + switch oType { + + case ObjectTypeVAR, ObjectTypeDOMAIN: + variable := &Variable{} + if dataType == "" { + return nil, fmt.Errorf("need data type") + } + dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse object type %v", err) + } + + // Get Attribute + dType := uint8(dataTypeUint) + attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) + + variable.Name = parameterName + variable.DataType = dType + variable.Attribute = attribute + variable.SubIndex = 0 + + if strings.Contains(defaultValue, "$NODEID") { + re := regexp.MustCompile(`\+?\$NODEID\+?`) + defaultValue = re.ReplaceAllString(defaultValue, "") + } else { + nodeId = 0 + } + variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) + if err != nil { + return nil, fmt.Errorf("failed to parse 'DefaultValue' for x%x|x%x, because %v (datatype :x%x)", "", 0, err, variable.DataType) + } + variable.value = make([]byte, len(variable.valueDefault)) + copy(variable.value, variable.valueDefault) + entry.object = variable + return nil, nil + + case ObjectTypeARRAY: + // Array objects do not allow holes in subindex numbers + // So pre-init slice up to subnumber + sub, err := strconv.ParseUint(subNumber, 0, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse subnumber %v", err) + } + vList := NewArray(uint8(sub)) + entry.object = vList + return vList, nil + + case ObjectTypeRECORD: + sub, err := strconv.ParseUint(subNumber, 0, 8) + if err != nil { + return nil, fmt.Errorf("failed to parse subnumber %v", err) + } + // Record objects allow holes in mapping + // Sub-objects will be added with "append" + vList := NewRecordWithLength(uint8(sub)) + entry.object = vList + return vList, nil + + default: + return nil, fmt.Errorf("unknown object type %v", oType) + } +} + +func populateSubEntry( + entry *Entry, + vlist *VariableList, + nodeId uint8, + parameterName string, + defaultValue string, + objectType string, + pdoMapping string, + lowLimit string, + highLimit string, + accessType string, + dataType string, + subIndex uint8, +) error { + if dataType == "" { + return fmt.Errorf("need data type") + } + dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) + if err != nil { + return fmt.Errorf("failed to parse object type %v", err) + } + + // Get Attribute + dType := uint8(dataTypeUint) + attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) + + variable := &Variable{ + Name: parameterName, + DataType: byte(dataTypeUint), + Attribute: attribute, + SubIndex: subIndex, + } + if strings.Contains(defaultValue, "$NODEID") { + re := regexp.MustCompile(`\+?\$NODEID\+?`) + defaultValue = re.ReplaceAllString(defaultValue, "") + } else { + nodeId = 0 + } + variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) + if err != nil { + return fmt.Errorf("failed to parse 'DefaultValue' %v %v %v", err, defaultValue, variable.DataType) + } + variable.value = make([]byte, len(variable.valueDefault)) + copy(variable.value, variable.valueDefault) + + switch entry.ObjectType { + case ObjectTypeARRAY, ObjectTypeRECORD: + vlist.Variables[subIndex] = variable + entry.subEntriesNameMap[parameterName] = subIndex + default: + return fmt.Errorf("add member not supported for ObjectType : %v", entry.ObjectType) + } + + return nil +} + +// Remove '\t' and ' ' characters at beginning +// and beginning of line +func trimSpaces(b []byte) []byte { + start, end := 0, len(b) + + for start < end && (b[start] == ' ' || b[start] == '\t') { + start++ + } + for end > start && (b[end-1] == ' ' || b[end-1] == '\t') { + end-- + } + return b[start:end] +} + +// Remove '\t' and ' ' characters at beginning +// and remove also '\r' at end of line +func trimSpacesAndr(b []byte) []byte { + start, end := 0, len(b) + + for start < end && (b[start] == ' ' || b[start] == '\t') { + start++ + } + for end > start && (b[end-1] == ' ' || b[end-1] == '\t' || b[end-1] == '\r') { + end-- + } + return b[start:end] +} diff --git a/pkg/od/variable.go b/pkg/od/variable.go index 25bf73f..312872b 100644 --- a/pkg/od/variable.go +++ b/pkg/od/variable.go @@ -21,199 +21,6 @@ func (variable *Variable) DefaultValue() []byte { return variable.valueDefault } -// func PopulateVariable( -// variable *Variable, -// nodeId uint8, -// parameterName string, -// defaultValue string, -// objectType string, -// pdoMapping string, -// lowLimit string, -// highLimit string, -// accessType string, -// dataType string, -// subNumber string, -// ) error { - -// if dataType == "" { -// return fmt.Errorf("need data type") -// } -// dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) -// if err != nil { -// return fmt.Errorf("failed to parse object type %v", err) -// } - -// // Get Attribute -// dType := uint8(dataTypeUint) -// attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) - -// variable.Name = parameterName -// variable.DataType = dType -// variable.Attribute = attribute -// variable.SubIndex = 0 - -// if strings.Contains(defaultValue, "$NODEID") { -// re := regexp.MustCompile(`\+?\$NODEID\+?`) -// defaultValue = re.ReplaceAllString(defaultValue, "") -// } else { -// nodeId = 0 -// } -// variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) -// if err != nil { -// return fmt.Errorf("failed to parse 'DefaultValue' for x%x|x%x, because %v (datatype :x%x)", "", 0, err, variable.DataType) -// } -// variable.value = make([]byte, len(variable.valueDefault)) -// copy(variable.value, variable.valueDefault) -// return nil -// } - -func PopulateEntry( - entry *Entry, - nodeId uint8, - parameterName string, - defaultValue string, - objectType string, - pdoMapping string, - lowLimit string, - highLimit string, - accessType string, - dataType string, - subNumber string, -) (*VariableList, error) { - - oType := uint8(0) - // Determine object type - // If no object type, default to 7 (CiA spec) - if objectType == "" { - oType = 7 - } else { - oTypeUint, err := strconv.ParseUint(objectType, 0, 8) - if err != nil { - return nil, fmt.Errorf("failed to parse object type %v", err) - } - oType = uint8(oTypeUint) - } - entry.ObjectType = oType - - // Add necessary stuff depending on oType - switch oType { - - case ObjectTypeVAR, ObjectTypeDOMAIN: - variable := &Variable{} - if dataType == "" { - return nil, fmt.Errorf("need data type") - } - dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) - if err != nil { - return nil, fmt.Errorf("failed to parse object type %v", err) - } - - // Get Attribute - dType := uint8(dataTypeUint) - attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) - - variable.Name = parameterName - variable.DataType = dType - variable.Attribute = attribute - variable.SubIndex = 0 - - if strings.Contains(defaultValue, "$NODEID") { - re := regexp.MustCompile(`\+?\$NODEID\+?`) - defaultValue = re.ReplaceAllString(defaultValue, "") - } else { - nodeId = 0 - } - variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) - if err != nil { - return nil, fmt.Errorf("failed to parse 'DefaultValue' for x%x|x%x, because %v (datatype :x%x)", "", 0, err, variable.DataType) - } - variable.value = make([]byte, len(variable.valueDefault)) - copy(variable.value, variable.valueDefault) - entry.object = variable - return nil, nil - - case ObjectTypeARRAY: - // Array objects do not allow holes in subindex numbers - // So pre-init slice up to subnumber - sub, err := strconv.ParseUint(subNumber, 0, 8) - if err != nil { - return nil, fmt.Errorf("failed to parse subnumber %v", err) - } - vList := NewArray(uint8(sub)) - entry.object = vList - return vList, nil - - case ObjectTypeRECORD: - // Record objects allow holes in mapping - // Sub-objects will be added with "append" - vList := NewRecord() - entry.object = vList - return vList, nil - - default: - return nil, fmt.Errorf("unknown object type %v", oType) - } -} - -func PopulateSubEntry( - entry *Entry, - vlist *VariableList, - nodeId uint8, - parameterName string, - defaultValue string, - objectType string, - pdoMapping string, - lowLimit string, - highLimit string, - accessType string, - dataType string, - subIndex uint8, -) error { - if dataType == "" { - return fmt.Errorf("need data type") - } - dataTypeUint, err := strconv.ParseUint(dataType, 0, 8) - if err != nil { - return fmt.Errorf("failed to parse object type %v", err) - } - - // Get Attribute - dType := uint8(dataTypeUint) - attribute := EncodeAttribute(accessType, pdoMapping == "1", dType) - - variable := &Variable{ - Name: parameterName, - DataType: byte(dataTypeUint), - Attribute: attribute, - SubIndex: 0, - } - if strings.Contains(defaultValue, "$NODEID") { - re := regexp.MustCompile(`\+?\$NODEID\+?`) - defaultValue = re.ReplaceAllString(defaultValue, "") - } else { - nodeId = 0 - } - variable.valueDefault, err = EncodeFromString(defaultValue, variable.DataType, nodeId) - if err != nil { - return fmt.Errorf("failed to parse 'DefaultValue' %v %v %v", err, defaultValue, variable.DataType) - } - variable.value = make([]byte, len(variable.valueDefault)) - copy(variable.value, variable.valueDefault) - - switch entry.ObjectType { - case ObjectTypeARRAY: - vlist.Variables[subIndex] = variable - entry.subEntriesNameMap[parameterName] = subIndex - case ObjectTypeRECORD: - vlist.Variables = append(vlist.Variables, variable) - entry.subEntriesNameMap[parameterName] = subIndex - default: - return fmt.Errorf("add member not supported for ObjectType : %v", entry.ObjectType) - } - - return nil -} - // Create variable from section entry func NewVariableFromSection( section *ini.Section, From bcc25edc0b2375aa0cefa8fe6056a2317c6bd905 Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 03/11] Changed : reverted to use bufio.Scanner, otherwise we can have subtleties with new lines depending on OS type... Changed : reverted change on record as we truly do not know the length of the underlying array Changed : od.Default() will use this second implementation, not all tests are passing yet --- pkg/od/parser_v2.go | 70 +++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/pkg/od/parser_v2.go b/pkg/od/parser_v2.go index c7707a8..b0fbc0d 100644 --- a/pkg/od/parser_v2.go +++ b/pkg/od/parser_v2.go @@ -1,6 +1,7 @@ package od import ( + "bufio" "bytes" "fmt" "io" @@ -33,23 +34,30 @@ import ( // - file i/o ==> not much to do here func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { - filename := file.(string) - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer f.Close() - + var err error bu := &bytes.Buffer{} - io.Copy(bu, f) - buffer := bu.Bytes() + + switch fType := file.(type) { + case string: + f, err := os.Open(fType) + if err != nil { + return nil, err + } + defer f.Close() + bu = &bytes.Buffer{} + io.Copy(bu, f) + + case []byte: + bu = bytes.NewBuffer(fType) + default: + return nil, fmt.Errorf("unsupported type") + } od := NewOD() var section string entry := &Entry{} vList := &VariableList{} - start := 0 isEntry := false isSubEntry := false subindex := uint8(0) @@ -64,16 +72,12 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { var accessType string var dataType string - // Scan hole EDS file - for i, b := range buffer { + scanner := bufio.NewScanner(bu) - if b != '\n' { - continue - } + for scanner.Scan() { // New line detected - lineRaw := buffer[start:i] - start = i + 1 + lineRaw := scanner.Bytes() // Skip if less than 2 chars if len(lineRaw) < 2 { @@ -88,8 +92,7 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { } // Handle section headers: [section] - if line[0] == '[' && line[len(line)-2] == ']' { - + if line[0] == '[' && line[len(line)-1] == ']' { // A section should be of length 4 at least if len(line) < 4 { continue @@ -141,7 +144,7 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { isEntry = false isSubEntry = false - sectionBytes := line[1 : len(line)-2] + sectionBytes := line[1 : len(line)-1] // Check if a sub entry or the actual entry // A subentry should be more than 4 bytes long @@ -194,7 +197,7 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { if equalsIdx := bytes.IndexByte(line, '='); equalsIdx != -1 { key := string(trimSpaces(line[:equalsIdx])) - value := string(trimSpacesAndr(line[equalsIdx+1:])) + value := string(trimSpaces(line[equalsIdx+1:])) // We will get the different elements of the entry switch key { @@ -300,13 +303,9 @@ func populateEntry( return vList, nil case ObjectTypeRECORD: - sub, err := strconv.ParseUint(subNumber, 0, 8) - if err != nil { - return nil, fmt.Errorf("failed to parse subnumber %v", err) - } // Record objects allow holes in mapping // Sub-objects will be added with "append" - vList := NewRecordWithLength(uint8(sub)) + vList := NewRecord() entry.object = vList return vList, nil @@ -361,9 +360,12 @@ func populateSubEntry( copy(variable.value, variable.valueDefault) switch entry.ObjectType { - case ObjectTypeARRAY, ObjectTypeRECORD: + case ObjectTypeARRAY: vlist.Variables[subIndex] = variable entry.subEntriesNameMap[parameterName] = subIndex + case ObjectTypeRECORD: + vlist.Variables = append(vlist.Variables, variable) + entry.subEntriesNameMap[parameterName] = subIndex default: return fmt.Errorf("add member not supported for ObjectType : %v", entry.ObjectType) } @@ -384,17 +386,3 @@ func trimSpaces(b []byte) []byte { } return b[start:end] } - -// Remove '\t' and ' ' characters at beginning -// and remove also '\r' at end of line -func trimSpacesAndr(b []byte) []byte { - start, end := 0, len(b) - - for start < end && (b[start] == ' ' || b[start] == '\t') { - start++ - } - for end > start && (b[end-1] == ' ' || b[end-1] == '\t' || b[end-1] == '\r') { - end-- - } - return b[start:end] -} From 7f5ac7cfdf37a239da2174aebb36422f99545124 Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 04/11] Added : create a read seeker for OD for creating in memory zip --- pkg/node/local.go | 8 ++++++-- pkg/od/od.go | 9 ++++++++- pkg/od/parser.go | 6 ++---- pkg/od/parser_v2.go | 1 + 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/node/local.go b/pkg/node/local.go index 4e7c8fb..27abc17 100644 --- a/pkg/node/local.go +++ b/pkg/node/local.go @@ -342,10 +342,10 @@ func NewLocalNode( switch format { case od.FormatEDSAscii: node.logger.Info("EDS is downloadable via object 0x1021 in ASCII format") - odict.AddReader(edsStore.Index, edsStore.Name, odict.Reader) + odict.AddReader(edsStore.Index, edsStore.Name, odict.NewReaderSeeker()) case od.FormatEDSZipped: node.logger.Info("EDS is downloadable via object 0x1021 in Zipped format") - compressed, err := createInMemoryZip("compressed.eds", odict.Reader) + compressed, err := createInMemoryZip("compressed.eds", odict.NewReaderSeeker()) if err != nil { node.logger.Error("failed to compress EDS", "error", err) return nil, err @@ -364,6 +364,10 @@ func NewLocalNode( // for example. func createInMemoryZip(filename string, r io.ReadSeeker) ([]byte, error) { + if r == nil { + return nil, fmt.Errorf("expecting a reader %v", r) + } + buffer := new(bytes.Buffer) zipWriter := zip.NewWriter(buffer) // Create a file inside the zip diff --git a/pkg/od/od.go b/pkg/od/od.go index 30430a7..840729d 100644 --- a/pkg/od/od.go +++ b/pkg/od/od.go @@ -1,6 +1,7 @@ package od import ( + "bytes" "fmt" "io" "log/slog" @@ -14,13 +15,19 @@ var _logger = slog.Default() // ObjectDictionary is used for storing all entries of a CANopen node // according to CiA 301. This is the internal representation of an EDS file type ObjectDictionary struct { - Reader io.ReadSeeker logger *slog.Logger + rawOd []byte iniFile *ini.File entriesByIndexValue map[uint16]*Entry entriesByIndexName map[string]*Entry } +// Create a new reader object for reading +// raw OD file. +func (od *ObjectDictionary) NewReaderSeeker() io.ReadSeeker { + return bytes.NewReader(od.rawOd) +} + // Add an entry to OD, any existing entry will be replaced func (od *ObjectDictionary) addEntry(entry *Entry) { _, entryIndexValueExists := od.entriesByIndexValue[entry.Index] diff --git a/pkg/od/parser.go b/pkg/od/parser.go index ffc5c2b..e1e4b0e 100644 --- a/pkg/od/parser.go +++ b/pkg/od/parser.go @@ -23,7 +23,7 @@ var matchSubidxRegExp = regexp.MustCompile(`^([0-9A-Fa-f]{4})sub([0-9A-Fa-f]+)$` // Return embeded default object dictionary func Default() *ObjectDictionary { - defaultOd, err := Parse(rawDefaultOd, 0) + defaultOd, err := ParseV2(rawDefaultOd, 0) if err != nil { panic(err) } @@ -48,9 +48,7 @@ func Parse(file any, nodeId uint8) (*ObjectDictionary, error) { // Write data from edsFile to the buffer // Don't care if fails _, _ = edsFile.WriteTo(&buf) - reader := bytes.NewReader(buf.Bytes()) - od.Reader = reader - od.iniFile = edsFile + od.rawOd = buf.Bytes() // Get all the sections in the file sections := edsFile.Sections() diff --git a/pkg/od/parser_v2.go b/pkg/od/parser_v2.go index b0fbc0d..0735ca7 100644 --- a/pkg/od/parser_v2.go +++ b/pkg/od/parser_v2.go @@ -54,6 +54,7 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { } od := NewOD() + od.rawOd = bu.Bytes() var section string entry := &Entry{} From de7c523e66e0a9a62fbfd54c2f15fb05556b2a4c Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 05/11] Changed : getting rid of intermediate .ini which isn't very useful --- pkg/od/export.go | 12 +++++++++++- pkg/od/od.go | 3 --- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/od/export.go b/pkg/od/export.go index a59c7bb..75f1761 100644 --- a/pkg/od/export.go +++ b/pkg/od/export.go @@ -2,6 +2,7 @@ package od import ( "fmt" + "io" "sort" "strconv" @@ -15,7 +16,16 @@ import ( // work for this library. func ExportEDS(odict *ObjectDictionary, defaultValues bool, filename string) error { if defaultValues { - return odict.iniFile.SaveTo(filename) + r := odict.NewReaderSeeker() + buffer, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("failed to read OD raw data %v", err) + } + i, err := ini.Load(buffer) + if err != nil { + return fmt.Errorf("failed to load .INI %v", err) + } + return i.SaveTo(filename) } eds := ini.Empty() diff --git a/pkg/od/od.go b/pkg/od/od.go index 840729d..3dce139 100644 --- a/pkg/od/od.go +++ b/pkg/od/od.go @@ -6,8 +6,6 @@ import ( "io" "log/slog" "sync" - - "gopkg.in/ini.v1" ) var _logger = slog.Default() @@ -17,7 +15,6 @@ var _logger = slog.Default() type ObjectDictionary struct { logger *slog.Logger rawOd []byte - iniFile *ini.File entriesByIndexValue map[uint16]*Entry entriesByIndexName map[string]*Entry } From 31b08a8e53e513da0d23ac9dd12a96406347d36f Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 06/11] Tests : adding benchmark for OD parser, new v2 parser is ~15x faster. Changed : removed unused variables --- pkg/od/parser.go | 4 - pkg/od/parser_test.go | 17 +++ pkg/od/parser_v2.go | 258 +++++++++++++++++++++++++++++------------- 3 files changed, 196 insertions(+), 83 deletions(-) diff --git a/pkg/od/parser.go b/pkg/od/parser.go index e1e4b0e..105149b 100644 --- a/pkg/od/parser.go +++ b/pkg/od/parser.go @@ -17,10 +17,6 @@ import ( var f embed.FS var rawDefaultOd []byte -// Get index & subindex matching -var matchIdxRegExp = regexp.MustCompile(`^[0-9A-Fa-f]{4}$`) -var matchSubidxRegExp = regexp.MustCompile(`^([0-9A-Fa-f]{4})sub([0-9A-Fa-f]+)$`) - // Return embeded default object dictionary func Default() *ObjectDictionary { defaultOd, err := ParseV2(rawDefaultOd, 0) diff --git a/pkg/od/parser_test.go b/pkg/od/parser_test.go index 0b1f87d..903a1d4 100644 --- a/pkg/od/parser_test.go +++ b/pkg/od/parser_test.go @@ -11,3 +11,20 @@ func TestParseDefault(t *testing.T) { od := Default() assert.NotNil(t, od) } + +func BenchmarkParser(b *testing.B) { + b.Run("od default parse", func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := Parse(rawDefaultOd, 0x10) + assert.Nil(b, err) + } + }) + + b.Run("od default parse v2", func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := ParseV2(rawDefaultOd, 0x10) + assert.Nil(b, err) + } + }) + +} diff --git a/pkg/od/parser_v2.go b/pkg/od/parser_v2.go index 0735ca7..d259379 100644 --- a/pkg/od/parser_v2.go +++ b/pkg/od/parser_v2.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "regexp" "strconv" "strings" ) @@ -55,8 +54,6 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { od := NewOD() od.rawOd = bu.Bytes() - - var section string entry := &Entry{} vList := &VariableList{} isEntry := false @@ -67,8 +64,6 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { var parameterName string var objectType string var pdoMapping string - var lowLimit string - var highLimit string var subNumber string var accessType string var dataType string @@ -101,45 +96,41 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { // New section, this means we have finished building // Previous one, so take all the values and update the section - if parameterName != "" && isEntry { - entry.Name = parameterName - od.entriesByIndexName[parameterName] = entry - vList, err = populateEntry( - entry, - nodeId, - parameterName, - defaultValue, - objectType, - pdoMapping, - lowLimit, - highLimit, - accessType, - dataType, - subNumber, - ) - - if err != nil { - return nil, fmt.Errorf("failed to create new entry %v", err) - } - - } else if parameterName != "" && isSubEntry { - err = populateSubEntry( - entry, - vList, - nodeId, - parameterName, - defaultValue, - objectType, - pdoMapping, - lowLimit, - highLimit, - accessType, - dataType, - subindex, - ) - - if err != nil { - return nil, fmt.Errorf("failed to create sub entry %v", err) + if parameterName != "" { + if isEntry { + entry.Name = parameterName + od.entriesByIndexName[parameterName] = entry + vList, err = populateEntry( + entry, + nodeId, + parameterName, + defaultValue, + objectType, + pdoMapping, + accessType, + dataType, + subNumber, + ) + + if err != nil { + return nil, fmt.Errorf("failed to create new entry %v", err) + } + } else if isSubEntry { + err = populateSubEntry( + entry, + vList, + nodeId, + parameterName, + defaultValue, + pdoMapping, + accessType, + dataType, + subindex, + ) + + if err != nil { + return nil, fmt.Errorf("failed to create sub entry %v", err) + } } } @@ -149,12 +140,9 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { // Check if a sub entry or the actual entry // A subentry should be more than 4 bytes long - subSection := sectionBytes[4:] - if len(subSection) < 4 && matchIdxRegExp.Match(sectionBytes) { - section = string(sectionBytes) + if isValidHex4(sectionBytes) { - // Add a new entry inside object dictionary - idx, err := strconv.ParseUint(section, 16, 16) + idx, err := hexAsciiToUint(sectionBytes) if err != nil { return nil, err } @@ -165,15 +153,14 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { entry.logger = od.logger od.entriesByIndexValue[uint16(idx)] = entry - } else if matchSubidxRegExp.Match(sectionBytes) { - section = string(sectionBytes) - // TODO we could get entry to double check if ever something is out of order - isSubEntry = true - // Subindex part is from the 7th letter onwards - sidx, err := strconv.ParseUint(section[7:], 16, 8) + } else if isValidSubIndexFormat(sectionBytes) { + + sidx, err := hexAsciiToUint(sectionBytes[7:]) if err != nil { return nil, err } + // TODO we could get entry to double check if ever something is out of order + isSubEntry = true subindex = uint8(sidx) } @@ -182,8 +169,6 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { parameterName = "" objectType = "" pdoMapping = "" - lowLimit = "" - highLimit = "" subNumber = "" accessType = "" dataType = "" @@ -203,27 +188,64 @@ func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { // We will get the different elements of the entry switch key { case "ParameterName": - parameterName = value + parameterName = string(value) case "ObjectType": - objectType = value + objectType = string(value) case "SubNumber": - subNumber = value + subNumber = string(value) case "AccessType": - accessType = value + accessType = string(value) case "DataType": - dataType = value - case "LowLimit": - lowLimit = value - case "HighLimit": - highLimit = value + dataType = string(value) case "DefaultValue": - defaultValue = value + defaultValue = string(value) case "PDOMapping": - pdoMapping = value + pdoMapping = string(value) + } + } + } + // Last index or subindex part + // New section, this means we have finished building + // Previous one, so take all the values and update the section + if parameterName != "" { + if isEntry { + entry.Name = parameterName + od.entriesByIndexName[parameterName] = entry + _, err = populateEntry( + entry, + nodeId, + parameterName, + defaultValue, + objectType, + pdoMapping, + accessType, + dataType, + subNumber, + ) + + if err != nil { + return nil, fmt.Errorf("failed to create new entry %v", err) + } + } else if isSubEntry { + err = populateSubEntry( + entry, + vList, + nodeId, + parameterName, + defaultValue, + pdoMapping, + accessType, + dataType, + subindex, + ) + + if err != nil { + return nil, fmt.Errorf("failed to create sub entry %v", err) } } } + return od, nil } @@ -234,8 +256,6 @@ func populateEntry( defaultValue string, objectType string, pdoMapping string, - lowLimit string, - highLimit string, accessType string, dataType string, subNumber string, @@ -277,9 +297,8 @@ func populateEntry( variable.Attribute = attribute variable.SubIndex = 0 - if strings.Contains(defaultValue, "$NODEID") { - re := regexp.MustCompile(`\+?\$NODEID\+?`) - defaultValue = re.ReplaceAllString(defaultValue, "") + if strings.Index(defaultValue, "$NODEID") != -1 { + defaultValue = fastRemoveNodeID(defaultValue) } else { nodeId = 0 } @@ -321,14 +340,12 @@ func populateSubEntry( nodeId uint8, parameterName string, defaultValue string, - objectType string, pdoMapping string, - lowLimit string, - highLimit string, accessType string, dataType string, subIndex uint8, ) error { + if dataType == "" { return fmt.Errorf("need data type") } @@ -347,9 +364,8 @@ func populateSubEntry( Attribute: attribute, SubIndex: subIndex, } - if strings.Contains(defaultValue, "$NODEID") { - re := regexp.MustCompile(`\+?\$NODEID\+?`) - defaultValue = re.ReplaceAllString(defaultValue, "") + if strings.Index(defaultValue, "$NODEID") != -1 { + defaultValue = fastRemoveNodeID(defaultValue) } else { nodeId = 0 } @@ -387,3 +403,87 @@ func trimSpaces(b []byte) []byte { } return b[start:end] } + +func hexAsciiToUint(bytes []byte) (uint64, error) { + var num uint64 + + for _, b := range bytes { + var digit uint64 + + switch { + case b >= '0' && b <= '9': + digit = uint64(b - '0') // Convert '0'-'9' to 0-9 + case b >= 'A' && b <= 'F': + digit = uint64(b - 'A' + 10) // Convert 'A'-'F' to 10-15 + case b >= 'a' && b <= 'f': + digit = uint64(b - 'a' + 10) // Convert 'a'-'f' to 10-15 + default: + return 0, fmt.Errorf("invalid hex character: %c", b) + } + + num = (num << 4) | digit // Left shift by 4 (multiply by 16) and add new digit + } + + return num, nil +} + +// Check if exactly 4 hex digits (no regex) +func isValidHex4(b []byte) bool { + if len(b) != 4 { + return false + } + for _, c := range b { + if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f')) { + return false + } + } + return true +} + +// Check if format is "XXXXsubYY" (without regex) +func isValidSubIndexFormat(b []byte) bool { + + // Must be at least "XXXXsubY" (4+3+1 chars) + if len(b) < 8 { + return false + } + // Check first 4 chars are hex + if !isValidHex4(b[:4]) { + return false + } + // Check "sub" part (fixed position) + if string(b[4:7]) != "sub" { + return false + } + // Check remaining are hex + for _, c := range b[7:] { + if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f')) { + return false + } + } + + return true +} + +func fastRemoveNodeID(s string) string { + b := make([]byte, 0, len(s)) // Preallocate same capacity as input string + + i := 0 + for i < len(s) { + if s[i] == '$' && len(s) > i+6 && s[i:i+7] == "$NODEID" { + i += 7 // Skip "$NODEID" + // Skip optional '+' after "$NODEID" + if i < len(s) && s[i] == '+' { + i++ + } + // Skip optional '+' before "$NODEID" + if len(b) > 0 && b[len(b)-1] == '+' { + b = b[:len(b)-1] + } + continue + } + b = append(b, s[i]) + i++ + } + return string(b) +} From ecdd2b93263195df2cc2ebd52ea0929c4bf11508 Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 07/11] Added : od.Parser type for parsing ODs Added : SetParser method to network for updating the way ODs are parsed. For now, we will stay with v1. --- pkg/network/network.go | 14 ++++++++++---- pkg/od/parser.go | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/network/network.go b/pkg/network/network.go index d581843..049e09c 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -42,8 +42,9 @@ type Network struct { *sdo.SDOClient controllers map[uint8]*n.NodeProcessor // Network has an its own SDOClient - odMap map[uint8]*ObjectDictionaryInformation - logger *slog.Logger + odMap map[uint8]*ObjectDictionaryInformation + odParser od.Parser + logger *slog.Logger } type ObjectDictionaryInformation struct { @@ -71,6 +72,7 @@ func NewNetwork(bus canopen.Bus) Network { controllers: map[uint8]*n.NodeProcessor{}, BusManager: canopen.NewBusManager(bus), odMap: map[uint8]*ObjectDictionaryInformation{}, + odParser: od.Parse, logger: slog.Default(), } } @@ -210,7 +212,7 @@ func (network *Network) CreateLocalNode(nodeId uint8, odict any) (*n.LocalNode, switch odType := odict.(type) { case string: - odNode, err = od.Parse(odType, nodeId) + odNode, err = network.odParser(odType, nodeId) if err != nil { return nil, err } @@ -268,7 +270,7 @@ func (network *Network) AddRemoteNode(nodeId uint8, odict any) (*n.RemoteNode, e switch odType := odict.(type) { case string: - odNode, err = od.Parse(odType, nodeId) + odNode, err = network.odParser(odType, nodeId) if err != nil { return nil, err } @@ -419,3 +421,7 @@ func (network *Network) Scan(timeoutMs uint32) (map[uint8]NodeInformation, error func (network *Network) SetLogger(logger *slog.Logger) { network.logger = logger } + +func (network *Network) SetParser(parser od.Parser) { + network.odParser = parser +} diff --git a/pkg/od/parser.go b/pkg/od/parser.go index 4d0ba2b..3b8dfd0 100644 --- a/pkg/od/parser.go +++ b/pkg/od/parser.go @@ -11,6 +11,8 @@ import ( "gopkg.in/ini.v1" ) +type Parser func(file any, nodeId uint8) (*ObjectDictionary, error) + // Parse an EDS file // file can be either a path or an *os.File or []byte // Other file types could be supported in the future From 487ff57c06b0045a3fcbcc70e7b9aaa92a730800 Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 08/11] Changed : some comments on parse v2 Changed : seperate file "base.go" for embedded OD. --- pkg/od/base.go | 17 +++++++++++++++++ pkg/od/parser.go | 15 --------------- pkg/od/parser_v2.go | 8 +++++--- 3 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 pkg/od/base.go diff --git a/pkg/od/base.go b/pkg/od/base.go new file mode 100644 index 0000000..d026886 --- /dev/null +++ b/pkg/od/base.go @@ -0,0 +1,17 @@ +package od + +import "embed" + +//go:embed base.eds + +var f embed.FS +var rawDefaultOd []byte + +// Return embeded default object dictionary +func Default() *ObjectDictionary { + defaultOd, err := ParseV2(rawDefaultOd, 0) + if err != nil { + panic(err) + } + return defaultOd +} diff --git a/pkg/od/parser.go b/pkg/od/parser.go index 105149b..4d0ba2b 100644 --- a/pkg/od/parser.go +++ b/pkg/od/parser.go @@ -3,7 +3,6 @@ package od import ( "archive/zip" "bytes" - "embed" "fmt" "io" "regexp" @@ -12,20 +11,6 @@ import ( "gopkg.in/ini.v1" ) -//go:embed base.eds - -var f embed.FS -var rawDefaultOd []byte - -// Return embeded default object dictionary -func Default() *ObjectDictionary { - defaultOd, err := ParseV2(rawDefaultOd, 0) - if err != nil { - panic(err) - } - return defaultOd -} - // Parse an EDS file // file can be either a path or an *os.File or []byte // Other file types could be supported in the future diff --git a/pkg/od/parser_v2.go b/pkg/od/parser_v2.go index d259379..5f8ffe0 100644 --- a/pkg/od/parser_v2.go +++ b/pkg/od/parser_v2.go @@ -10,7 +10,7 @@ import ( "strings" ) -// v2 of OD parser, this implementation is ~10x faster +// v2 of OD parser, this implementation is ~15x faster // than the previous one but has some caveats : // // - it expects OD definitions to be "in order" i.e. @@ -25,12 +25,13 @@ import ( // ... // [1001] // +// With the current OD architecture, only minor other +// optimizations could be done. // The remaining bottlenecks are the following : // -// - regexp are pretty slow, not sure if would could do much better // - bytes to string conversions for values create a lot of unnecessary allocation. // As values are mostly stored in bytes anyway, we could remove this step. -// - file i/o ==> not much to do here +// - bufio.Scanner() ==> more performant implementation ? func ParseV2(file any, nodeId uint8) (*ObjectDictionary, error) { var err error @@ -465,6 +466,7 @@ func isValidSubIndexFormat(b []byte) bool { return true } +// Remove "$NODEID" from given string func fastRemoveNodeID(s string) string { b := make([]byte, 0, len(s)) // Preallocate same capacity as input string From 61d3ce5565b4c6b628f5805cf8e4a93eb484836c Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:49 +0100 Subject: [PATCH 09/11] Changed : bump go.mod&go.sum --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a0d7c4c..61d029a 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/samsamfire/gocanopen go 1.22 require ( - github.com/stretchr/testify v1.8.4 - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 + github.com/stretchr/testify v1.9.0 + golang.org/x/sys v0.21.0 gopkg.in/ini.v1 v1.67.0 ) diff --git a/go.sum b/go.sum index e7c289b..39eb5da 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= From c143aec8eaf184ed5251b87a3c1bc7737a3bfb45 Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:50 +0100 Subject: [PATCH 10/11] Added : od.Parser type for parsing ODs Added : SetParser method to network for updating the way ODs are parsed. For now, we will stay with v1. --- pkg/network/network.go | 14 ++++++++++---- pkg/od/parser.go | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/network/network.go b/pkg/network/network.go index d581843..049e09c 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -42,8 +42,9 @@ type Network struct { *sdo.SDOClient controllers map[uint8]*n.NodeProcessor // Network has an its own SDOClient - odMap map[uint8]*ObjectDictionaryInformation - logger *slog.Logger + odMap map[uint8]*ObjectDictionaryInformation + odParser od.Parser + logger *slog.Logger } type ObjectDictionaryInformation struct { @@ -71,6 +72,7 @@ func NewNetwork(bus canopen.Bus) Network { controllers: map[uint8]*n.NodeProcessor{}, BusManager: canopen.NewBusManager(bus), odMap: map[uint8]*ObjectDictionaryInformation{}, + odParser: od.Parse, logger: slog.Default(), } } @@ -210,7 +212,7 @@ func (network *Network) CreateLocalNode(nodeId uint8, odict any) (*n.LocalNode, switch odType := odict.(type) { case string: - odNode, err = od.Parse(odType, nodeId) + odNode, err = network.odParser(odType, nodeId) if err != nil { return nil, err } @@ -268,7 +270,7 @@ func (network *Network) AddRemoteNode(nodeId uint8, odict any) (*n.RemoteNode, e switch odType := odict.(type) { case string: - odNode, err = od.Parse(odType, nodeId) + odNode, err = network.odParser(odType, nodeId) if err != nil { return nil, err } @@ -419,3 +421,7 @@ func (network *Network) Scan(timeoutMs uint32) (map[uint8]NodeInformation, error func (network *Network) SetLogger(logger *slog.Logger) { network.logger = logger } + +func (network *Network) SetParser(parser od.Parser) { + network.odParser = parser +} diff --git a/pkg/od/parser.go b/pkg/od/parser.go index 4d0ba2b..3b8dfd0 100644 --- a/pkg/od/parser.go +++ b/pkg/od/parser.go @@ -11,6 +11,8 @@ import ( "gopkg.in/ini.v1" ) +type Parser func(file any, nodeId uint8) (*ObjectDictionary, error) + // Parse an EDS file // file can be either a path or an *os.File or []byte // Other file types could be supported in the future From 7595cfb9f867ba2921ef21799ba207744f14386d Mon Sep 17 00:00:00 2001 From: Samuel Lee Date: Mon, 13 Jan 2025 00:51:50 +0100 Subject: [PATCH 11/11] Changed : bump go.mod&go.sum --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a0d7c4c..61d029a 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/samsamfire/gocanopen go 1.22 require ( - github.com/stretchr/testify v1.8.4 - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 + github.com/stretchr/testify v1.9.0 + golang.org/x/sys v0.21.0 gopkg.in/ini.v1 v1.67.0 ) diff --git a/go.sum b/go.sum index e7c289b..39eb5da 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=