diff --git a/mos/build.go b/mos/build.go index d6ad703d..a5a51dbe 100644 --- a/mos/build.go +++ b/mos/build.go @@ -54,7 +54,6 @@ var ( buildDryRunFlag = flag.Bool("build-dry-run", false, "do not actually run the build, only prepare") buildParamsFlag = flag.String("build-params", "", "build params file") buildTarget = flag.String("build-target", moscommon.BuildTargetDefault, "target to build with make") - keepTempFiles = flag.Bool("keep-temp-files", false, "keep temp files after the build is done (by default they are in ~/.mos/tmp)") modules = flag.StringArray("module", []string{}, "location of the module from mos.yaml, in the format: \"module_name:/path/to/location\". Can be used multiple times.") libs = flag.StringArray("lib", []string{}, "location of the lib from mos.yaml, in the format: \"lib_name:/path/to/location\". Can be used multiple times.") libsUpdateInterval = flag.Duration("libs-update-interval", time.Hour*1, "how often to update already fetched libs") diff --git a/mos/build_remote.go b/mos/build_remote.go index 24e1a67c..e0a2c1e5 100644 --- a/mos/build_remote.go +++ b/mos/build_remote.go @@ -28,18 +28,19 @@ import ( "path/filepath" "strings" + "github.com/cesanta/errors" + "github.com/golang/glog" "github.com/mongoose-os/mos/common/ourfilepath" "github.com/mongoose-os/mos/common/ourio" - "github.com/mongoose-os/mos/mos/ourutil" "github.com/mongoose-os/mos/mos/build" "github.com/mongoose-os/mos/mos/build/archive" moscommon "github.com/mongoose-os/mos/mos/common" "github.com/mongoose-os/mos/mos/common/paths" + "github.com/mongoose-os/mos/mos/flags" "github.com/mongoose-os/mos/mos/interpreter" "github.com/mongoose-os/mos/mos/manifest_parser" + "github.com/mongoose-os/mos/mos/ourutil" "github.com/mongoose-os/mos/mos/version" - "github.com/cesanta/errors" - "github.com/golang/glog" yaml "gopkg.in/yaml.v2" ) @@ -62,7 +63,7 @@ func buildRemote(bParams *buildParams) error { if err != nil { return errors.Trace(err) } - if !*keepTempFiles { + if !*flags.KeepTempFiles { defer os.RemoveAll(appStagingDir) } if *verbose { diff --git a/mos/flags/flags.go b/mos/flags/flags.go index 8eb56e8d..fd9a7f46 100644 --- a/mos/flags/flags.go +++ b/mos/flags/flags.go @@ -22,8 +22,8 @@ import ( "io/ioutil" "time" - "github.com/mongoose-os/mos/mos/ourutil" "github.com/cesanta/errors" + "github.com/mongoose-os/mos/mos/ourutil" flag "github.com/spf13/pflag" ) @@ -86,6 +86,8 @@ var ( Subject = flag.String("subject", "", "Subject for CSR or certificate") GDBServerCmd = flag.String("gdb-server-cmd", "/usr/local/bin/serve_core.py", "") + + KeepTempFiles = flag.Bool("keep-temp-files", false, "keep temp files after the build is done (by default they are in ~/.mos/tmp)") ) func Platform() string { diff --git a/mos/flash.go b/mos/flash.go index cd474ac2..36bb0986 100644 --- a/mos/flash.go +++ b/mos/flash.go @@ -27,8 +27,9 @@ import ( "context" + "github.com/cesanta/errors" + "github.com/golang/glog" "github.com/mongoose-os/mos/common/fwbundle" - "github.com/mongoose-os/mos/mos/ourutil" "github.com/mongoose-os/mos/mos/dev" "github.com/mongoose-os/mos/mos/devutil" "github.com/mongoose-os/mos/mos/flags" @@ -38,9 +39,8 @@ import ( espFlasher "github.com/mongoose-os/mos/mos/flash/esp/flasher" "github.com/mongoose-os/mos/mos/flash/rs14100" "github.com/mongoose-os/mos/mos/flash/stm32" + "github.com/mongoose-os/mos/mos/ourutil" "github.com/mongoose-os/mos/mos/version" - "github.com/cesanta/errors" - "github.com/golang/glog" flag "github.com/spf13/pflag" ) @@ -146,7 +146,7 @@ func flash(ctx context.Context, devConn dev.DevConn) error { if err != nil { return errors.Annotatef(err, "failed to load %s", fwname) } - if !*keepTempFiles { + if !*flags.KeepTempFiles { defer fw.Cleanup() } diff --git a/mos/update/ubuntu.go b/mos/update/ubuntu.go new file mode 100644 index 00000000..30d527a4 --- /dev/null +++ b/mos/update/ubuntu.go @@ -0,0 +1,145 @@ +// +// Copyright (c) 2014-2019 Cesanta Software Limited +// All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package update + +import ( + "bufio" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/cesanta/errors" + "github.com/golang/glog" + "github.com/mongoose-os/mos/mos/flags" + "github.com/mongoose-os/mos/mos/ourutil" + "github.com/mongoose-os/mos/mos/version" +) + +var ( + ubuntuPackageNames = map[UpdateChannel]string{ + UpdateChannelRelease: "mos", + UpdateChannelLatest: "mos-latest", + } + ubuntuRepoName = "ppa:mongoose-os/mos" + ubuntuDEBURLPrefix = "https://mongoose-os.com/downloads/debs/" + ubuntuDEBArchiveURL = ubuntuDEBURLPrefix + "index.txt" + ubuntuDEBNameRegexp = regexp.MustCompile(`^(?P[^_]+)_(?P[^+]+)\+(?P[^~]+)\~(?P[^\d]+)\d+_(?P[^.]+)\.deb`) +) + +func doUbuntuUpdateRepo(oldUpdChannel, newUpdChannel UpdateChannel) error { + oldPkg := ubuntuPackageNames[oldUpdChannel] + newPkg := ubuntuPackageNames[newUpdChannel] + + // Start with an apt-get update. + // Do not fail because some unrelated repo may be screwed up + // but our PPA might still be ok. + ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "update") + + // Check if mos and mos-latest are among the available packages. + output, err := ourutil.GetCommandOutput("apt-cache", "showpkg", newPkg) + if err != nil { + return errors.Annotatef(err, "faild to get package info") + } + if !strings.Contains(output, "/lists/") { + // No package info in repo lists - we should (re-)add our repo. + // This can happen, for example, after release upgrade which disables 3rd-party repos. + if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-add-repository", "-y", "-u", ubuntuRepoName); err != nil { + return errors.Trace(err) + } + } + + if oldPkg != newPkg { + if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "remove", "-y", oldPkg); err != nil { + return errors.Trace(err) + } + } + return ourutil.RunCmd(ourutil.CmdOutAlways, "sudo", "apt-get", "install", "-y", newPkg) +} + +func doUbuntuUpdateDEB(oldUpdChannel UpdateChannel, newMosVersion string) error { + distr := version.GetUbuntuBuildIDParts(version.BuildId)["distr"] + arch := runtime.GOARCH + if arch == "arm" { + arch = "armhf" + } + glog.Infof("Distr: %s, arch: %s", distr, arch) + debName := "" + { + glog.Infof("Fetching package archive index from %s...", ubuntuDEBArchiveURL) + resp, err := http.Get(ubuntuDEBArchiveURL) + if err != nil { + return errors.Annotatef(err, "failed to fetch package archive index") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return errors.Errorf("failed to fetch package archive index from %s: %s", ubuntuDEBArchiveURL, resp.Status) + } + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + d := strings.Split(scanner.Text(), " ")[0] + p := ourutil.FindNamedSubmatches(ubuntuDEBNameRegexp, scanner.Text()) + glog.V(2).Infof("%s -> %s", scanner.Text(), p) + if p["distr"] == distr && p["version"] == newMosVersion && p["arch"] == arch { + debName = d + break + } + } + if debName == "" { + return errors.Errorf("no package found for %s %s %s", newMosVersion, distr, arch) + } + } + debURL := ubuntuDEBURLPrefix + debName + ourutil.Reportf("Downloading %s...", debURL) + resp, err := http.Get(debURL) + if err != nil { + return errors.Annotatef(err, "failed to fetch package") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return errors.Errorf("failed to fetch package from %s: %s", debURL, resp.Status) + } + debData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Annotatef(err, "failed to fetch package") + } + tempDir, err := ioutil.TempDir("", "mosdeb-") + if err != nil { + return errors.Annotatef(err, "failed to create staging directory") + } + if !*flags.KeepTempFiles { + defer os.RemoveAll(tempDir) + } + debFile := filepath.Join(tempDir, debName) + if err = ioutil.WriteFile(debFile, debData, 0644); err != nil { + return errors.Annotatef(err, "failed to write package file") + } + glog.Infof("Wrote %s %s", debFile, resp.Status) + ourutil.Reportf("Removing installed package (if any)...") + oldPkg := ubuntuPackageNames[oldUpdChannel] + ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "remove", "-y", oldPkg) + ourutil.Reportf("Installing new package...") + if err = ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "dpkg", "-i", debFile); err != nil { + return errors.Annotatef(err, "failed to install %s", debFile) + } + ourutil.Reportf("Package installed") + return nil +} diff --git a/mos/update/update.go b/mos/update/update.go index 1b1c4bea..87ff9813 100644 --- a/mos/update/update.go +++ b/mos/update/update.go @@ -63,11 +63,6 @@ var ( UpdateChannelRelease: "mos", UpdateChannelLatest: "mos-latest", } - ubuntuPackageNames = map[UpdateChannel]string{ - UpdateChannelRelease: "mos", - UpdateChannelLatest: "mos-latest", - } - ubuntuRepoName = "ppa:mongoose-os/mos" ) // mosVersion can be either exact mos version like "1.6", or update channel @@ -108,36 +103,6 @@ func GetServerMosVersion(mosVersion string, extraInfo ...string) (*version.Versi return &serverVersion, nil } -func doUbuntuUpdate(oldUpdChannel, newUpdChannel UpdateChannel) error { - oldPkg := ubuntuPackageNames[oldUpdChannel] - newPkg := ubuntuPackageNames[newUpdChannel] - - // Start with an apt-get update. - // Do not fail because some unrelated repo may be screwed up - // but our PPA might still be ok. - ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "update") - - // Check if mos and mos-latest are among the available packages. - output, err := ourutil.GetCommandOutput("apt-cache", "showpkg", newPkg) - if err != nil { - return errors.Annotatef(err, "faild to get package info") - } - if !strings.Contains(output, "/lists/") { - // No package info in repo lists - we should (re-)add our repo. - // This can happen, for example, after release upgrade which disables 3rd-party repos. - if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-add-repository", "-y", "-u", ubuntuRepoName); err != nil { - return errors.Trace(err) - } - } - - if oldPkg != newPkg { - if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "remove", "-y", oldPkg); err != nil { - return errors.Trace(err) - } - } - return ourutil.RunCmd(ourutil.CmdOutAlways, "sudo", "apt-get", "install", "-y", newPkg) -} - func doBrewUpdate(oldUpdChannel, newUpdChannel UpdateChannel) error { oldPkg := brewPackageNames[oldUpdChannel] newPkg := brewPackageNames[newUpdChannel] @@ -174,7 +139,11 @@ func Update(ctx context.Context, devConn dev.DevConn) error { } if version.LooksLikeUbuntuBuildId(version.BuildId) { - return doUbuntuUpdate(updChannel, newUpdChannel) + if newMosVersion == string(newUpdChannel) { + return doUbuntuUpdateRepo(updChannel, newUpdChannel) + } else { + return doUbuntuUpdateDEB(updChannel, newMosVersion) + } } else if version.LooksLikeBrewBuildId(version.BuildId) { return doBrewUpdate(updChannel, newUpdChannel) } else if version.LooksLikeDistrBuildId(version.BuildId) { diff --git a/mos/version/version_util.go b/mos/version/version_util.go index 72b6f9b8..1417f7d2 100644 --- a/mos/version/version_util.go +++ b/mos/version/version_util.go @@ -42,9 +42,7 @@ const ( var ( regexpVersionNumber = regexp.MustCompile(`^\d+\.[0-9.]*$`) - regexpBuildIdDistr = regexp.MustCompile( - `^(?P[^+]+)\+(?P[^~]+)\~(?P.+)$`, - ) + regexpBuildIdDistr = regexp.MustCompile(`^(?P[^+]+)\+(?P[^~]+)\~(?P[^\d]+)\d+$`) ubuntuDistrNames = []string{"xenial", "bionic", "cosmic", "disco"} ) @@ -85,21 +83,26 @@ func LooksLikeBrewBuildId(s string) bool { // debian build id, returns either "latest" or "release". Otherwise, returns // an empty string. func GetUbuntuUpdateChannel(buildId string) string { - matches := ourutil.FindNamedSubmatches(regexpBuildIdDistr, buildId) - if matches != nil { - for _, v := range ubuntuDistrNames { - if strings.HasPrefix(matches["distr"], v) { - if LooksLikeVersionNumber(matches["version"]) { - return "release" - } else { - return "latest" - } + parts := GetUbuntuBuildIDParts(buildId) + if parts == nil { + return "" + } + for _, v := range ubuntuDistrNames { + if strings.HasPrefix(parts["distr"], v) { + if LooksLikeVersionNumber(parts["version"]) { + return "release" + } else { + return "latest" } } } return "" } +func GetUbuntuBuildIDParts(buildId string) map[string]string { + return ourutil.FindNamedSubmatches(regexpBuildIdDistr, buildId) +} + func GetUserAgent() string { return fmt.Sprintf("mos/%s %s (%s; %s)", Version, BuildId, runtime.GOOS, runtime.GOARCH) }