diff --git a/src/cmd/singularity/cli/action_flags.go b/src/cmd/singularity/cli/action_flags.go index 5491a479fb..aa3171eb59 100644 --- a/src/cmd/singularity/cli/action_flags.go +++ b/src/cmd/singularity/cli/action_flags.go @@ -15,6 +15,7 @@ import ( // actionflags.go contains flag variables for action-like commands to draw from var ( + AppName string BindPaths []string HomePath string OverlayPath []string @@ -74,6 +75,9 @@ func init() { // initPathVars initializes flags that take a string argument func initPathVars() { + // --app + actionFlags.StringVar(&AppName, "app", "", "Set container app to run") + // -B|--bind actionFlags.StringSliceVarP(&BindPaths, "bind", "B", []string{}, "A user-bind path specification. spec has the format src[:dest[:opts]], where src and dest are outside and inside paths. If dest is not given, it is set equal to src. Mount options ('opts') may be specified as 'ro' (read-only) or 'rw' (read/write, which is the default). Multiple bind paths can be given by a comma separated list.") actionFlags.SetAnnotation("bind", "argtag", []string{""}) diff --git a/src/cmd/singularity/cli/actions.go b/src/cmd/singularity/cli/actions.go index 8f664c8cee..df1ab57630 100644 --- a/src/cmd/singularity/cli/actions.go +++ b/src/cmd/singularity/cli/actions.go @@ -77,6 +77,7 @@ func init() { cmd.Flags().AddFlag(actionFlags.Lookup("no-init")) cmd.Flags().AddFlag(actionFlags.Lookup("security")) cmd.Flags().AddFlag(actionFlags.Lookup("apply-cgroups")) + cmd.Flags().AddFlag(actionFlags.Lookup("app")) cmd.Flags().SetInterspersed(false) } @@ -431,6 +432,8 @@ func execStarter(cobraCmd *cobra.Command, image string, args []string, name stri Env := []string{sylog.GetEnvVar(), "SRUNTIME=singularity"} + generator.AddProcessEnv("SINGULARITY_APPNAME", AppName) + cfg := &config.Common{ EngineName: singularity.Name, ContainerID: name, diff --git a/src/cmd/singularity/cli/singularity.go b/src/cmd/singularity/cli/singularity.go index 2f2d5c448c..b76f0a3bf2 100644 --- a/src/cmd/singularity/cli/singularity.go +++ b/src/cmd/singularity/cli/singularity.go @@ -17,6 +17,7 @@ import ( "github.com/sylabs/singularity/src/docs" "github.com/sylabs/singularity/src/pkg/buildcfg" "github.com/sylabs/singularity/src/pkg/sylog" + "github.com/sylabs/singularity/src/pkg/syplugin" "github.com/sylabs/singularity/src/pkg/util/auth" ) @@ -153,6 +154,7 @@ func handleEnv(flag *pflag.Flag) { func persistentPreRun(cmd *cobra.Command, args []string) { setSylogMessageLevel(cmd, args) updateFlagsFromEnv(cmd) + syplugin.Init() } // sylabsToken process the authentication Token diff --git a/src/pkg/build/assemblers/assembler_sif.go b/src/pkg/build/assemblers/assembler_sif.go index f35984e3e6..f56380c8f2 100644 --- a/src/pkg/build/assemblers/assembler_sif.go +++ b/src/pkg/build/assemblers/assembler_sif.go @@ -17,6 +17,7 @@ import ( "github.com/satori/go.uuid" "github.com/sylabs/sif/pkg/sif" "github.com/sylabs/singularity/src/pkg/build/types" + "github.com/sylabs/singularity/src/pkg/build/types/parser" "github.com/sylabs/singularity/src/pkg/sylog" ) @@ -87,7 +88,7 @@ func (a *SIFAssembler) Assemble(b *types.Bundle, path string) (err error) { // convert definition to plain text var buf bytes.Buffer - b.Recipe.WriteDefinitionFile(&buf) + parser.WriteDefinitionFile(&(b.Recipe), &buf) def := buf.Bytes() // make system partition image diff --git a/src/pkg/build/build.go b/src/pkg/build/build.go index e039ff00ea..7cc6c0d91e 100644 --- a/src/pkg/build/build.go +++ b/src/pkg/build/build.go @@ -21,8 +21,10 @@ import ( "github.com/sylabs/singularity/src/pkg/build/assemblers" "github.com/sylabs/singularity/src/pkg/build/sources" "github.com/sylabs/singularity/src/pkg/build/types" + "github.com/sylabs/singularity/src/pkg/build/types/parser" "github.com/sylabs/singularity/src/pkg/buildcfg" "github.com/sylabs/singularity/src/pkg/sylog" + "github.com/sylabs/singularity/src/pkg/syplugin" syexec "github.com/sylabs/singularity/src/pkg/util/exec" "github.com/sylabs/singularity/src/runtime/engines/config" "github.com/sylabs/singularity/src/runtime/engines/config/oci" @@ -173,6 +175,9 @@ func (b *Build) Full() error { } } + syplugin.BuildHandleBundles(b.b) + b.b.Recipe.BuildData.Post += syplugin.BuildHandlePosts() + if hasScripts(b.d) { if syscall.Getuid() == 0 { sylog.Debugf("Starting build engine") @@ -377,43 +382,41 @@ func getcp(def types.Definition, libraryURL, authToken string) (ConveyorPacker, // makeDef gets a definition object from a spec func makeDef(spec string) (types.Definition, error) { - var def types.Definition - if ok, err := IsValidURI(spec); ok && err == nil { // URI passed as spec - def, err = types.NewDefinitionFromURI(spec) - if err != nil { - return def, fmt.Errorf("unable to parse URI %s: %v", spec, err) - } - } else if _, err := os.Stat(spec); err == nil { - // Non-URI passed as spec + return types.NewDefinitionFromURI(spec) + } + + // Non-URI passed as spec + ok, err := parser.IsValidDefinition(spec) + if ok { + sylog.Debugf("Found valid definition: %s\n", spec) + // File exists and contains valid definition defFile, err := os.Open(spec) if err != nil { - return def, fmt.Errorf("unable to open file %s: %v", spec, err) + return types.Definition{}, fmt.Errorf("unable to open file %s: %v", spec, err) } defer defFile.Close() - if d, err := types.ParseDefinitionFile(defFile); err == nil { - // must be root to build from a definition - if os.Getuid() != 0 { - sylog.Fatalf("You must be the root user to build from a Singularity recipe file") - } - //definition used as input - def = d - } else { - //local image or sandbox, make sure it exists on filesystem - def = types.Definition{ - Header: map[string]string{ - "bootstrap": "localimage", - "from": spec, - }, - } + // must be root to build from a definition + if os.Getuid() != 0 { + sylog.Fatalf("You must be the root user to build from a Singularity recipe file") } - } else { - return def, fmt.Errorf("unable to build from %s: %v", spec, err) + + return parser.ParseDefinitionFile(defFile) + } else if err == nil { + // File exists and does NOT contain a valid definition + // local image or sandbox + return types.Definition{ + Header: map[string]string{ + "bootstrap": "localimage", + "from": spec, + }, + }, nil } - return def, nil + // File does NOT exist or cannot be opened for another reason + return types.Definition{}, fmt.Errorf("unable to build from %s: %v", spec, err) } func (b *Build) addOptions() { @@ -545,7 +548,7 @@ func insertDefinition(b *types.Bundle) error { return err } - b.Recipe.WriteDefinitionFile(f) + parser.WriteDefinitionFile(&b.Recipe, f) return nil } diff --git a/src/pkg/build/sources/conveyorPacker_arch_test.go b/src/pkg/build/sources/conveyorPacker_arch_test.go index 905f99d3bb..05c0f66a64 100644 --- a/src/pkg/build/sources/conveyorPacker_arch_test.go +++ b/src/pkg/build/sources/conveyorPacker_arch_test.go @@ -12,6 +12,7 @@ import ( "github.com/sylabs/singularity/src/pkg/build/sources" "github.com/sylabs/singularity/src/pkg/build/types" + "github.com/sylabs/singularity/src/pkg/build/types/parser" "github.com/sylabs/singularity/src/pkg/test" ) @@ -41,7 +42,7 @@ func TestArchConveyor(t *testing.T) { return } - b.Recipe, err = types.ParseDefinitionFile(defFile) + b.Recipe, err = parser.ParseDefinitionFile(defFile) if err != nil { t.Fatalf("failed to parse definition file %s: %v\n", archDef, err) } @@ -75,7 +76,7 @@ func TestArchPacker(t *testing.T) { return } - b.Recipe, err = types.ParseDefinitionFile(defFile) + b.Recipe, err = parser.ParseDefinitionFile(defFile) if err != nil { t.Fatalf("failed to parse definition file %s: %v\n", archDef, err) } diff --git a/src/pkg/build/sources/conveyorPacker_busybox_test.go b/src/pkg/build/sources/conveyorPacker_busybox_test.go index 51dd477ac7..d16f8b6630 100644 --- a/src/pkg/build/sources/conveyorPacker_busybox_test.go +++ b/src/pkg/build/sources/conveyorPacker_busybox_test.go @@ -11,6 +11,7 @@ import ( "github.com/sylabs/singularity/src/pkg/build/sources" "github.com/sylabs/singularity/src/pkg/build/types" + "github.com/sylabs/singularity/src/pkg/build/types/parser" "github.com/sylabs/singularity/src/pkg/test" ) @@ -36,7 +37,7 @@ func TestBusyBoxConveyor(t *testing.T) { return } - b.Recipe, err = types.ParseDefinitionFile(defFile) + b.Recipe, err = parser.ParseDefinitionFile(defFile) if err != nil { t.Fatalf("failed to parse definition file %s: %v\n", busyBoxDef, err) } @@ -66,7 +67,7 @@ func TestBusyBoxPacker(t *testing.T) { return } - b.Recipe, err = types.ParseDefinitionFile(defFile) + b.Recipe, err = parser.ParseDefinitionFile(defFile) if err != nil { t.Fatalf("failed to parse definition file %s: %v\n", busyBoxDef, err) } diff --git a/src/pkg/build/sources/conveyorPacker_yum_test.go b/src/pkg/build/sources/conveyorPacker_yum_test.go index c0635dbbad..2f1090ca84 100644 --- a/src/pkg/build/sources/conveyorPacker_yum_test.go +++ b/src/pkg/build/sources/conveyorPacker_yum_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/sylabs/singularity/src/pkg/build/types" + "github.com/sylabs/singularity/src/pkg/build/types/parser" "github.com/sylabs/singularity/src/pkg/test" ) @@ -41,7 +42,7 @@ func TestYumConveyor(t *testing.T) { return } - b.Recipe, err = types.ParseDefinitionFile(defFile) + b.Recipe, err = parser.ParseDefinitionFile(defFile) if err != nil { t.Fatalf("failed to parse definition file %s: %v\n", yumDef, err) } @@ -77,7 +78,7 @@ func TestYumPacker(t *testing.T) { return } - b.Recipe, err = types.ParseDefinitionFile(defFile) + b.Recipe, err = parser.ParseDefinitionFile(defFile) if err != nil { t.Fatalf("failed to parse definition file %s: %v\n", yumDef, err) } diff --git a/src/pkg/build/types/definition.go b/src/pkg/build/types/definition.go index 64b9258ea5..b9d3537adf 100644 --- a/src/pkg/build/types/definition.go +++ b/src/pkg/build/types/definition.go @@ -92,30 +92,3 @@ func NewDefinitionFromJSON(r io.Reader) (d Definition, err error) { return d, nil } - -// validSections just contains a list of all the valid sections a definition file -// could contain. If any others are found, an error will generate -var validSections = map[string]bool{ - "help": true, - "setup": true, - "files": true, - "labels": true, - "environment": true, - "pre": true, - "post": true, - "runscript": true, - "test": true, - "startscript": true, -} - -// validHeaders just contains a list of all the valid headers a definition file -// could contain. If any others are found, an error will generate -var validHeaders = map[string]bool{ - "bootstrap": true, - "from": true, - "includecmd": true, - "mirrorurl": true, - "updateurl": true, - "osversion": true, - "include": true, -} diff --git a/src/pkg/build/types/definition_parser_deffile.go b/src/pkg/build/types/parser/deffile.go similarity index 68% rename from src/pkg/build/types/definition_parser_deffile.go rename to src/pkg/build/types/parser/deffile.go index 60ce2165e4..1946b183e3 100644 --- a/src/pkg/build/types/definition_parser_deffile.go +++ b/src/pkg/build/types/parser/deffile.go @@ -3,7 +3,7 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package types +package parser import ( "bufio" @@ -12,11 +12,15 @@ import ( "fmt" "io" "log" + "os" "reflect" "strings" + "sync" "unicode" + "github.com/sylabs/singularity/src/pkg/build/types" "github.com/sylabs/singularity/src/pkg/sylog" + "github.com/sylabs/singularity/src/pkg/syplugin" ) // scanDefinitionFile is the SplitFunc for the scanner that will parse the deffile. It will split into tokens @@ -57,14 +61,9 @@ func scanDefinitionFile(data []byte, atEOF bool) (advance int, token []byte, err // Check if the first word starts with % sign if word != nil && word[0] == '%' { // If the word starts with %, it's a section identifier - _, ok := validSections[string(word[1:])] // Validate that the section identifier is valid - if !ok { - // Invalid Section Identifier - return 0, nil, fmt.Errorf("invalid section identifier found: %s", string(word)) - } - - // Valid Section Identifier + // We no longer check if the word is a valid section identifier here, since we want to move to + // a more modular approach where we can parse arbitrary sections if inSection { // Here we found the end of the section return advance, retbuf.Bytes(), nil @@ -72,7 +71,7 @@ func scanDefinitionFile(data []byte, atEOF bool) (advance int, token []byte, err // When advance == 0 and we found a section identifier, that means we have already // parsed the header out and left the % as the first character in the data. This means // we can now parse into sections. - retbuf.Write(word[1:]) + retbuf.Write(line[1:]) retbuf.WriteString("\n") inSection = true } else { @@ -101,48 +100,109 @@ func scanDefinitionFile(data []byte, atEOF bool) (advance int, token []byte, err } -func insertSection(b []byte, sections map[string]string) { - for i := 0; i < len(b); i++ { - if b[i] == '\n' { - sections[string(b[:i])] = strings.TrimRightFunc(string(b[i+1:]), unicode.IsSpace) - break - } +func isValidSection(key string) bool { + if _, ok := validSections[key]; !ok { + return false + } + + return true +} + +func getSectionName(line string) string { + lineSplit := strings.SplitN(strings.ToLower(line), " ", 2) + + return lineSplit[0] +} + +// splitToken splits tok -> identline & content pair (sep on \n) +func splitToken(tok string) (ident string, content string) { + tokSplit := strings.SplitN(tok, "\n", 2) + if len(tokSplit) == 1 { + content = "" + } else { + content = tokSplit[1] } + + return strings.ToLower(tokSplit[0]), content + } -func doSections(s *bufio.Scanner, d *Definition) (err error) { - sections := make(map[string]string) +var sectionsMutex = &sync.Mutex{} + +// parseTokenSection splits the token into maximum 2 strings separated by a newline, +// and then inserts the section into the sections map +// +// goroutine safe +func parseTokenSection(tok string, sections map[string]string) { + split := strings.SplitN(tok, "\n", 2) + if len(split) != 2 { + return + } - token := strings.TrimSpace(s.Text()) + key := getSectionName(split[0]) + if !isValidSection(key) { + return + } + + sectionsMutex.Lock() + sections[key] = strings.TrimRightFunc(split[1], unicode.IsSpace) + sectionsMutex.Unlock() +} + +func doSections(s *bufio.Scanner, d *types.Definition) error { + sectionsMap := make(map[string]string) + + var wg sync.WaitGroup + + tok := strings.TrimSpace(s.Text()) //check if first thing parsed is a header or just a section - if strings.ToLower(token[0:9]) == "bootstrap" { - if err = doHeader(token, d); err != nil { + if strings.ToLower(tok[0:9]) == "bootstrap" { + if err := doHeader(tok, d); err != nil { sylog.Warningf("failed to parse DefFile header: %v\n", err) - return + return err } } else { //this is a section - insertSection([]byte(token), sections) + parseTokenSection(tok, sectionsMap) + syplugin.BuildHandleSections(splitToken(tok)) } //parse remaining sections while scanner can advance for s.Scan() { - if err = s.Err(); err != nil { - return + if err := s.Err(); err != nil { + return err } - b := s.Bytes() - insertSection(b, sections) + tok := s.Text() + + // Parse each token -> section + wg.Add(1) + go func() { + defer wg.Done() + parseTokenSection(tok, sectionsMap) + }() + + // Process any custom section handling + wg.Add(1) + go func() { + defer wg.Done() + syplugin.BuildHandleSections(splitToken(tok)) + }() } - if err = s.Err(); err != nil { - return + if err := s.Err(); err != nil { + return err } + wg.Wait() + return populateDefinition(sectionsMap, d) +} + +func populateDefinition(sections map[string]string, d *types.Definition) error { // Files are parsed as a map[string]string filesSections := strings.TrimSpace(sections["files"]) subs := strings.Split(filesSections, "\n") - var files []FileTransport + var files []types.FileTransport for _, line := range subs { @@ -159,7 +219,7 @@ func doSections(s *bufio.Scanner, d *Definition) (err error) { dst = strings.TrimSpace(lineSubs[1]) } - files = append(files, FileTransport{src, dst}) + files = append(files, types.FileTransport{Src: src, Dst: dst}) } // labels are parsed as a map[string]string @@ -184,8 +244,8 @@ func doSections(s *bufio.Scanner, d *Definition) (err error) { labels[key] = val } - d.ImageData = ImageData{ - ImageScripts: ImageScripts{ + d.ImageData = types.ImageData{ + ImageScripts: types.ImageScripts{ Help: sections["help"], Environment: sections["environment"], Runscript: sections["runscript"], @@ -195,7 +255,7 @@ func doSections(s *bufio.Scanner, d *Definition) (err error) { Labels: labels, } d.BuildData.Files = files - d.BuildData.Scripts = Scripts{ + d.BuildData.Scripts = types.Scripts{ Pre: sections["pre"], Setup: sections["setup"], Post: sections["post"], @@ -203,17 +263,17 @@ func doSections(s *bufio.Scanner, d *Definition) (err error) { } // make sure information was valid by checking if definition is not equal to an empty one - emptyDef := new(Definition) + emptyDef := new(types.Definition) // labels is always initialized emptyDef.Labels = make(map[string]string) if reflect.DeepEqual(d, emptyDef) { return fmt.Errorf("parsed definition did not have any valid information") } - return + return nil } -func doHeader(h string, d *Definition) (err error) { +func doHeader(h string, d *types.Definition) (err error) { h = strings.TrimSpace(h) toks := strings.Split(h, "\n") d.Header = make(map[string]string) @@ -240,7 +300,7 @@ func doHeader(h string, d *Definition) (err error) { // ParseDefinitionFile recieves a reader from a definition file // and parse it into a Definition struct or return error if // the definition file has a bad section. -func ParseDefinitionFile(r io.Reader) (d Definition, err error) { +func ParseDefinitionFile(r io.Reader) (d types.Definition, err error) { s := bufio.NewScanner(r) s.Split(scanDefinitionFile) @@ -262,7 +322,7 @@ func ParseDefinitionFile(r io.Reader) (d Definition, err error) { } func canGetHeader(r io.Reader) (ok bool, err error) { - var d Definition + var d types.Definition s := bufio.NewScanner(r) s.Split(scanDefinitionFile) @@ -279,7 +339,7 @@ func canGetHeader(r io.Reader) (ok bool, err error) { if err = doHeader(s.Text(), &d); err != nil { sylog.Warningf("failed to parse DefFile header: %v\n", err) - return + return false, nil } return true, nil @@ -295,7 +355,7 @@ func writeSectionIfExists(w io.Writer, ident string, s string) { } } -func writeFilesIfExists(w io.Writer, f []FileTransport) { +func writeFilesIfExists(w io.Writer, f []types.FileTransport) { if len(f) > 0 { @@ -335,7 +395,7 @@ func writeLabelsIfExists(w io.Writer, l map[string]string) { // WriteDefinitionFile is a helper func to output a Definition struct // into a definition file. -func (d *Definition) WriteDefinitionFile(w io.Writer) { +func WriteDefinitionFile(d *types.Definition, w io.Writer) { for k, v := range d.Header { w.Write([]byte(k)) w.Write([]byte(": ")) @@ -356,3 +416,42 @@ func (d *Definition) WriteDefinitionFile(w io.Writer) { writeSectionIfExists(w, "setup", d.BuildData.Setup) writeSectionIfExists(w, "post", d.BuildData.Post) } + +// IsValidDefinition returns whether or not the given file is a valid definition +func IsValidDefinition(source string) (valid bool, err error) { + defFile, err := os.Open(source) + if err != nil { + return false, err + } + defer defFile.Close() + + ok, _ := canGetHeader(defFile) + + return ok, nil +} + +// validSections just contains a list of all the valid sections a definition file +// could contain. If any others are found, an error will generate +var validSections = map[string]bool{ + "help": true, + "setup": true, + "files": true, + "labels": true, + "environment": true, + "pre": true, + "post": true, + "runscript": true, + "test": true, + "startscript": true, +} + +// validHeaders just contains a list of all the valid headers a definition file +// could contain. If any others are found, an error will generate +var validHeaders = map[string]bool{ + "bootstrap": true, + "from": true, + "includecmd": true, + "mirrorurl": true, + "osversion": true, + "include": true, +} diff --git a/src/pkg/build/types/definition_parser_deffile_test.go b/src/pkg/build/types/parser/deffile_test.go similarity index 59% rename from src/pkg/build/types/definition_parser_deffile_test.go rename to src/pkg/build/types/parser/deffile_test.go index 77b433b5c4..49ab3fa491 100644 --- a/src/pkg/build/types/definition_parser_deffile_test.go +++ b/src/pkg/build/types/parser/deffile_test.go @@ -3,7 +3,7 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package types +package parser import ( "bufio" @@ -13,6 +13,7 @@ import ( "reflect" "testing" + "github.com/sylabs/singularity/src/pkg/build/types" "github.com/sylabs/singularity/src/pkg/test" ) @@ -22,14 +23,14 @@ func TestScanDefinitionFile(t *testing.T) { defPath string sections string }{ - {"Arch", "../testdata_good/arch/arch", "../testdata_good/arch/arch_sections.json"}, - {"BusyBox", "../testdata_good/busybox/busybox", "../testdata_good/busybox/busybox_sections.json"}, - {"Debootstrap", "../testdata_good/debootstrap/debootstrap", "../testdata_good/debootstrap/debootstrap_sections.json"}, - {"Docker", "../testdata_good/docker/docker", "../testdata_good/docker/docker_sections.json"}, - {"LocalImage", "../testdata_good/localimage/localimage", "../testdata_good/localimage/localimage_sections.json"}, - {"Shub", "../testdata_good/shub/shub", "../testdata_good/shub/shub_sections.json"}, - {"Yum", "../testdata_good/yum/yum", "../testdata_good/yum/yum_sections.json"}, - {"Zypper", "../testdata_good/zypper/zypper", "../testdata_good/zypper/zypper_sections.json"}, + {"Arch", "../../testdata_good/arch/arch", "../../testdata_good/arch/arch_sections.json"}, + {"BusyBox", "../../testdata_good/busybox/busybox", "../../testdata_good/busybox/busybox_sections.json"}, + {"Debootstrap", "../../testdata_good/debootstrap/debootstrap", "../../testdata_good/debootstrap/debootstrap_sections.json"}, + {"Docker", "../../testdata_good/docker/docker", "../../testdata_good/docker/docker_sections.json"}, + {"LocalImage", "../../testdata_good/localimage/localimage", "../../testdata_good/localimage/localimage_sections.json"}, + {"Shub", "../../testdata_good/shub/shub", "../../testdata_good/shub/shub_sections.json"}, + {"Yum", "../../testdata_good/yum/yum", "../../testdata_good/yum/yum_sections.json"}, + {"Zypper", "../../testdata_good/zypper/zypper", "../../testdata_good/zypper/zypper_sections.json"}, } for _, tt := range tests { @@ -75,14 +76,14 @@ func TestParseDefinitionFile(t *testing.T) { defPath string jsonPath string }{ - {"Docker", "../testdata_good/docker/docker", "../testdata_good/docker/docker.json"}, - {"BusyBox", "../testdata_good/busybox/busybox", "../testdata_good/busybox/busybox.json"}, - {"Debootstrap", "../testdata_good/debootstrap/debootstrap", "../testdata_good/debootstrap/debootstrap.json"}, - {"Arch", "../testdata_good/arch/arch", "../testdata_good/arch/arch.json"}, - {"LocalImage", "../testdata_good/localimage/localimage", "../testdata_good/localimage/localimage.json"}, - {"Shub", "../testdata_good/shub/shub", "../testdata_good/shub/shub.json"}, - {"Yum", "../testdata_good/yum/yum", "../testdata_good/yum/yum.json"}, - {"Zypper", "../testdata_good/zypper/zypper", "../testdata_good/zypper/zypper.json"}, + {"Arch", "../../testdata_good/arch/arch", "../../testdata_good/arch/arch.json"}, + {"BusyBox", "../../testdata_good/busybox/busybox", "../../testdata_good/busybox/busybox.json"}, + {"Debootstrap", "../../testdata_good/debootstrap/debootstrap", "../../testdata_good/debootstrap/debootstrap.json"}, + {"Docker", "../../testdata_good/docker/docker", "../../testdata_good/docker/docker.json"}, + {"LocalImage", "../../testdata_good/localimage/localimage", "../../testdata_good/localimage/localimage.json"}, + {"Shub", "../../testdata_good/shub/shub", "../../testdata_good/shub/shub.json"}, + {"Yum", "../../testdata_good/yum/yum", "../../testdata_good/yum/yum.json"}, + {"Zypper", "../../testdata_good/zypper/zypper", "../../testdata_good/zypper/zypper.json"}, } for _, tt := range tests { @@ -104,7 +105,7 @@ func TestParseDefinitionFile(t *testing.T) { t.Fatal("failed to parse definition file:", err) } - var defCorrect Definition + var defCorrect types.Definition if err := json.NewDecoder(jsonFile).Decode(&defCorrect); err != nil { t.Fatal("failed to parse JSON:", err) } @@ -121,10 +122,10 @@ func TestParseDefinitionFileFailure(t *testing.T) { name string defPath string }{ - {"BadSection", "../testdata_bad/bad_section"}, - {"JSONInput1", "../testdata_bad/json_input_1"}, - {"JSONInput2", "../testdata_bad/json_input_2"}, - {"Empty", "../testdata_bad/empty"}, + //{"BadSection", "../../testdata_bad/bad_section"}, + {"JSONInput1", "../../testdata_bad/json_input_1"}, + {"JSONInput2", "../../testdata_bad/json_input_2"}, + {"Empty", "../../testdata_bad/empty"}, } for _, tt := range tests { diff --git a/src/pkg/syplugin/build.go b/src/pkg/syplugin/build.go new file mode 100644 index 0000000000..d81ca267dc --- /dev/null +++ b/src/pkg/syplugin/build.go @@ -0,0 +1,117 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the URIs of this project regarding your +// rights to use or distribute this software. + +package syplugin + +import ( + "fmt" + "sync" + + "github.com/sylabs/singularity/src/pkg/build/types" + "github.com/sylabs/singularity/src/pkg/sylog" +) + +var registeredBuildPlugins BuildPluginRegistry + +func init() { + registeredBuildPlugins = BuildPluginRegistry{ + Plugins: make(map[string]BuildPlugin), + } +} + +// BasePluginRegistry ... +type BasePluginRegistry struct { + sync.Mutex +} + +// BuildPluginRegistry ... +type BuildPluginRegistry struct { + BasePluginRegistry + Plugins map[string]BuildPlugin +} + +// RegisterBuildPlugin adds the plugin to the known plugins +func RegisterBuildPlugin(_pl interface{}) error { + pl, ok := _pl.(BuildPlugin) + if !ok { + return nil + } + + registeredBuildPlugins.Lock() + defer registeredBuildPlugins.Unlock() + + if _, ok := registeredBuildPlugins.Plugins[pl.Name()]; ok { + return fmt.Errorf("plugin name already registered: %s", pl.Name()) + } + + registeredBuildPlugins.Plugins[pl.Name()] = pl + return nil +} + +// GetBuildPlugins returns the list of known plugins +func GetBuildPlugins() map[string]BuildPlugin { + registeredBuildPlugins.Lock() + defer registeredBuildPlugins.Unlock() + + return registeredBuildPlugins.Plugins +} + +// BuildHandleSections runs the HandleSection() hook on every plugin +func BuildHandleSections(i, s string) { + var plwait sync.WaitGroup + + for name, pl := range GetBuildPlugins() { + plwait.Add(1) + go func(name string, pl BuildPlugin) { + defer plwait.Done() + sylog.Debugf("Running %s plugin: HandleSection() hook", name) + + pl.HandleSection(i, s) + }(name, pl) + } + + plwait.Wait() +} + +// BuildHandleBundles runs the HandleBundle() hook on every plugin +func BuildHandleBundles(b *types.Bundle) { + var plwait sync.WaitGroup + + for name, pl := range GetBuildPlugins() { + plwait.Add(1) + go func(name string, pl BuildPlugin) { + defer plwait.Done() + sylog.Debugf("Running %s plugin: HandleBundle() hook", name) + + pl.HandleBundle(b) + }(name, pl) + } + + plwait.Wait() +} + +// BuildHandlePosts runs the HandleBundle() hook on every plugin +func BuildHandlePosts() (ret string) { + for name, pl := range GetBuildPlugins() { + sylog.Debugf("Running %s plugin: HandlePost() hook", name) + + ret += pl.HandlePost() + } + + return +} + +// BuildPlugin is the interface for plugins on the build system +type BuildPlugin interface { + Name() string + HandleSection(string, string) + HandleBundle(*types.Bundle) + HandlePost() string +} + +// func init() { +// appPl := apps.New() +// RegisterBuildPlugin(&appPl) +// } diff --git a/src/pkg/syplugin/load.go b/src/pkg/syplugin/load.go new file mode 100644 index 0000000000..7e51eb8a5e --- /dev/null +++ b/src/pkg/syplugin/load.go @@ -0,0 +1,102 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the URIs of this project regarding your +// rights to use or distribute this software. + +package syplugin + +import ( + "fmt" + "path/filepath" + "plugin" + "sync" + + "github.com/sylabs/singularity/src/pkg/buildcfg" + "github.com/sylabs/singularity/src/pkg/sylog" + "github.com/sylabs/singularity/src/plugins/apps" +) + +type pluginRegisterFn func(interface{}) error + +var pluginRegisterFuncs = map[string]pluginRegisterFn{ + "BuildPlugin": RegisterBuildPlugin, +} + +func loadPlugins(pattern string) (pls []*plugin.Plugin, err error) { + paths, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + + for _, path := range paths { + pl, err := plugin.Open(path) + if err != nil { + return nil, err + } + + pls = append(pls, pl) + } + + return pls, nil +} + +func initPlugin(_pl *plugin.Plugin) error { + _new, err := _pl.Lookup("New") + if err != nil { + return err + } + + new, ok := _new.(func() interface{}) + if !ok { + return fmt.Errorf("Unable to get plugin new symbol") + } + pl := new() + registerPlugin(pl) + + return nil +} + +func registerPlugin(pl interface{}) { + var regWait sync.WaitGroup + + for plType, regFn := range pluginRegisterFuncs { + regWait.Add(1) + go func(plType string, regFn pluginRegisterFn) { + sylog.Debugf("Registering plugin as type %s", plType) + + if err := regFn(pl); err != nil { + sylog.Fatalf("Unable to register plugin??") + } + regWait.Done() + }(plType, regFn) + } + + regWait.Wait() +} + +// InitDynamic initializes plugins via dynamic loading. This is implemented but not +// fully featured, so we're using a static methodology until 3.1 +func InitDynamic() { + var plLoadWait sync.WaitGroup + pls, err := loadPlugins(filepath.Join(buildcfg.LIBDIR, "singularity/plugin/*")) + if err != nil { + sylog.Fatalf("Unable to load plugins from dir: %s", err) + } + + for _, pl := range pls { + plLoadWait.Add(1) + go func(pl *plugin.Plugin) { + defer plLoadWait.Done() + if err := initPlugin(pl); err != nil { + sylog.Fatalf("Something went wrong: %s", err) + } + }(pl) + } + + plLoadWait.Wait() +} + +// Init initializes plugins via static linking +func Init() { + registerPlugin(apps.New()) +} diff --git a/src/plugins/apps/apps.go b/src/plugins/apps/apps.go new file mode 100644 index 0000000000..0d2c772a31 --- /dev/null +++ b/src/plugins/apps/apps.go @@ -0,0 +1,318 @@ +// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the URIs of this project regarding your +// rights to use or distribute this software. + +// Package apps [apps-plugin] provides the functions which are necessary for adding SCI-F apps support +// to Singularity 3.0.0. In 3.1.0+, this package will be able to be built standalone as +// a plugin so it will be maintainable separately from the core Singularity functionality +package apps + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/sylabs/singularity/src/pkg/build/types" + "github.com/sylabs/singularity/src/pkg/sylog" +) + +const name = "singularity_apps" + +const ( + sectionInstall = "appinstall" + sectionFiles = "appfiles" + sectionEnv = "appenv" + sectionTest = "apptest" + sectionHelp = "apphelp" + sectionRun = "apprun" +) + +const ( + globalEnv94Base = `## App Global Exports For: %[1]s + +SCIF_APPDATA_%[1]s=/scif/data/%[1]s +SCIF_APPMETA_%[1]s=/scif/apps/%[1]s/scif +SCIF_APPROOT_%[1]s=/scif/apps/%[1]s +SCIF_APPBIN_%[1]s=/scif/apps/%[1]s/bin +SCIF_APPLIB_%[1]s=/scif/apps/%[1]s/lib + +export SCIF_APPDATA_%[1]s SCIF_APPMETA_%[1]s SCIF_APPROOT_%[1]s SCIF_APPBIN_%[1]s SCIF_APPLIB_%[1]s +` + + globalEnv94AppEnv = `export SCIF_APPENV_%[1]s="/scif/apps/%[1]s/scif/env/90-environment.sh" +` + globalEnv94AppRun = `export SCIF_APPRUN_%[1]s="/scif/apps/%[1]s/scif/runscript" +` + + scifEnv01Base = `#!/bin/sh + +SCIF_APPNAME=%[1]s +SCIF_APPROOT="/scif/apps/%[1]s" +SCIF_APPMETA="/scif/apps/%[1]s/scif" +SCIF_DATA="/scif/data" +SCIF_APPDATA="/scif/data/%[1]s" +SCIF_APPINPUT="/scif/data/%[1]s/input" +SCIF_APPOUTPUT="/scif/data/%[1]s/output" +export SCIF_APPDATA SCIF_APPNAME SCIF_APPROOT SCIF_APPMETA SCIF_APPINPUT SCIF_APPOUTPUT SCIF_DATA +` + + scifRunscriptBase = `#!/bin/sh + +%s +` + + scifInstallBase = ` +cd / +. %[1]s/env/01-base.sh + +cd %[1]s +%[2]s + +cd / +` +) + +// App stores the deffile sections of the app +type App struct { + Name string + Install string + Files string + Env string + Test string + Help string + Run string +} + +// BuildPlugin is the type which the build system can understand +type BuildPlugin struct { + Apps map[string]*App `json:"appsDefined"` + sync.Mutex +} + +// New returns a new BuildPlugin for the plugin registry to hold +func New() interface{} { + return &BuildPlugin{ + Apps: make(map[string]*App), + } + +} + +// Name returns this handler's name [singularity_apps] +func (pl *BuildPlugin) Name() string { + return name +} + +// HandleSection receives a string of each section from the deffile +func (pl *BuildPlugin) HandleSection(ident, section string) { + name, sect := getAppAndSection(ident) + if name == "" || sect == "" { + return + } + + pl.initApp(name) + app := pl.Apps[name] + + switch sect { + case sectionInstall: + app.Install = section + case sectionFiles: + app.Files = section + case sectionEnv: + app.Env = section + case sectionTest: + app.Test = section + case sectionHelp: + app.Help = section + case sectionRun: + app.Run = section + default: + return + } +} + +func (pl *BuildPlugin) initApp(name string) { + pl.Lock() + defer pl.Unlock() + + _, ok := pl.Apps[name] + if !ok { + pl.Apps[name] = &App{ + Name: name, + Install: "", + Files: "", + Env: "", + Test: "", + Help: "", + Run: "", + } + } +} + +// getAppAndSection returns the app name and section name from the header of the section: +// %SECTION APP ... returns APP, SECTION +func getAppAndSection(ident string) (appName string, sectionName string) { + identSplit := strings.Split(ident, " ") + + if len(identSplit) < 2 { + return "", "" + } + + return identSplit[1], identSplit[0] +} + +// HandleBundle is a hook where we can modify the bundle +func (pl *BuildPlugin) HandleBundle(b *types.Bundle) { + if err := pl.createAllApps(b); err != nil { + sylog.Fatalf("Unable to create apps: %s", err) + } +} + +func (pl *BuildPlugin) createAllApps(b *types.Bundle) error { + globalEnv94 := "" + + for name, app := range pl.Apps { + sylog.Debugf("Creating %s app in bundle", name) + if err := createAppRoot(b, app); err != nil { + return err + } + + if err := writeEnvFile(b, app); err != nil { + return err + } + + if err := writeRunscriptFile(b, app); err != nil { + return err + } + + if err := writeHelpFile(b, app); err != nil { + return err + } + + globalEnv94 += globalAppEnv(b, app) + } + + return writeFile(filepath.Join(b.Rootfs(), "/.singularity.d/env/94-appsbase.sh"), 0755, globalEnv94) +} + +func createAppRoot(b *types.Bundle, a *App) error { + if err := os.MkdirAll(appBase(b, a), 0755); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Join(appBase(b, a), "/scif/"), 0755); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Join(appBase(b, a), "/bin/"), 0755); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Join(appBase(b, a), "/lib/"), 0755); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Join(appBase(b, a), "/scif/env/"), 0755); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Join(appData(b, a), "/input/"), 0755); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Join(appData(b, a), "/output/"), 0755); err != nil { + return err + } + + return nil +} + +// %appenv and 01-base.sh +func writeEnvFile(b *types.Bundle, a *App) error { + content := fmt.Sprintf(scifEnv01Base, a.Name) + if err := writeFile(filepath.Join(appMeta(b, a), "/env/01-base.sh"), 0755, content); err != nil { + return err + } + + if a.Env == "" { + return nil + } + + return writeFile(filepath.Join(appMeta(b, a), "/env/90-environment.sh"), 0755, a.Env) +} + +func globalAppEnv(b *types.Bundle, a *App) string { + content := fmt.Sprintf(globalEnv94Base, a.Name) + + if _, err := os.Stat(filepath.Join(appMeta(b, a), "/env/90-environment.sh")); err == nil { + content += fmt.Sprintf(globalEnv94AppEnv, a.Name) + } + + if _, err := os.Stat(filepath.Join(appMeta(b, a), "/runscript")); err == nil { + content += fmt.Sprintf(globalEnv94AppRun, a.Name) + } + + return content +} + +// %apprun +func writeRunscriptFile(b *types.Bundle, a *App) error { + if a.Run == "" { + return nil + } + + content := fmt.Sprintf(scifRunscriptBase, a.Run) + return writeFile(filepath.Join(appMeta(b, a), "/runscript"), 0755, content) +} + +// %apphelp +func writeHelpFile(b *types.Bundle, a *App) error { + if a.Help == "" { + return nil + } + + return writeFile(filepath.Join(appMeta(b, a), "/runscript.help"), 0755, a.Help) +} + +//util funcs + +func appBase(b *types.Bundle, a *App) string { + return filepath.Join(b.Rootfs(), "/scif/apps/", a.Name) +} + +func appMeta(b *types.Bundle, a *App) string { + return filepath.Join(appBase(b, a), "/scif/") +} + +func appData(b *types.Bundle, a *App) string { + return filepath.Join(b.Rootfs(), "/scif/data/", a.Name) +} + +func writeFile(path string, perm os.FileMode, s string) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + defer f.Close() + + _, err = f.WriteString(s) + return err +} + +// HandlePost returns a script that should run after %post +func (pl *BuildPlugin) HandlePost() string { + post := "" + for name, app := range pl.Apps { + sylog.Debugf("Building app[%s] post script section", name) + + post += buildPost(app) + } + + return post +} + +func buildPost(a *App) string { + return fmt.Sprintf(scifInstallBase, filepath.Join("/scif/apps/", a.Name, "/scif"), a.Install) +}