Skip to content

Commit

Permalink
Allow mos updating to a specific version of Ubuntu package
Browse files Browse the repository at this point in the history
PPA only stores the latest version, this utilizes our own package archive

CL: mos: Allow mos updating to a specific version of Ubuntu package
  • Loading branch information
rojer committed May 8, 2019
1 parent fa488ce commit db6f2c4
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 58 deletions.
1 change: 0 additions & 1 deletion mos/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ var (
buildDryRunFlag = flag.Bool("build-dry-run", false, "do not actually run the build, only prepare")
buildParamsFlag = flag.String("build-params", "", "build params file")
buildTarget = flag.String("build-target", moscommon.BuildTargetDefault, "target to build with make")
keepTempFiles = flag.Bool("keep-temp-files", false, "keep temp files after the build is done (by default they are in ~/.mos/tmp)")
modules = flag.StringArray("module", []string{}, "location of the module from mos.yaml, in the format: \"module_name:/path/to/location\". Can be used multiple times.")
libs = flag.StringArray("lib", []string{}, "location of the lib from mos.yaml, in the format: \"lib_name:/path/to/location\". Can be used multiple times.")
libsUpdateInterval = flag.Duration("libs-update-interval", time.Hour*1, "how often to update already fetched libs")
Expand Down
9 changes: 5 additions & 4 deletions mos/build_remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,19 @@ import (
"path/filepath"
"strings"

"github.com/cesanta/errors"
"github.com/golang/glog"
"github.com/mongoose-os/mos/common/ourfilepath"
"github.com/mongoose-os/mos/common/ourio"
"github.com/mongoose-os/mos/mos/ourutil"
"github.com/mongoose-os/mos/mos/build"
"github.com/mongoose-os/mos/mos/build/archive"
moscommon "github.com/mongoose-os/mos/mos/common"
"github.com/mongoose-os/mos/mos/common/paths"
"github.com/mongoose-os/mos/mos/flags"
"github.com/mongoose-os/mos/mos/interpreter"
"github.com/mongoose-os/mos/mos/manifest_parser"
"github.com/mongoose-os/mos/mos/ourutil"
"github.com/mongoose-os/mos/mos/version"
"github.com/cesanta/errors"
"github.com/golang/glog"
yaml "gopkg.in/yaml.v2"
)

Expand All @@ -62,7 +63,7 @@ func buildRemote(bParams *buildParams) error {
if err != nil {
return errors.Trace(err)
}
if !*keepTempFiles {
if !*flags.KeepTempFiles {
defer os.RemoveAll(appStagingDir)
}
if *verbose {
Expand Down
4 changes: 3 additions & 1 deletion mos/flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ import (
"io/ioutil"
"time"

"github.com/mongoose-os/mos/mos/ourutil"
"github.com/cesanta/errors"
"github.com/mongoose-os/mos/mos/ourutil"
flag "github.com/spf13/pflag"
)

Expand Down Expand Up @@ -86,6 +86,8 @@ var (
Subject = flag.String("subject", "", "Subject for CSR or certificate")

GDBServerCmd = flag.String("gdb-server-cmd", "/usr/local/bin/serve_core.py", "")

KeepTempFiles = flag.Bool("keep-temp-files", false, "keep temp files after the build is done (by default they are in ~/.mos/tmp)")
)

func Platform() string {
Expand Down
8 changes: 4 additions & 4 deletions mos/flash.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ import (

"context"

"github.com/cesanta/errors"
"github.com/golang/glog"
"github.com/mongoose-os/mos/common/fwbundle"
"github.com/mongoose-os/mos/mos/ourutil"
"github.com/mongoose-os/mos/mos/dev"
"github.com/mongoose-os/mos/mos/devutil"
"github.com/mongoose-os/mos/mos/flags"
Expand All @@ -38,9 +39,8 @@ import (
espFlasher "github.com/mongoose-os/mos/mos/flash/esp/flasher"
"github.com/mongoose-os/mos/mos/flash/rs14100"
"github.com/mongoose-os/mos/mos/flash/stm32"
"github.com/mongoose-os/mos/mos/ourutil"
"github.com/mongoose-os/mos/mos/version"
"github.com/cesanta/errors"
"github.com/golang/glog"
flag "github.com/spf13/pflag"
)

Expand Down Expand Up @@ -146,7 +146,7 @@ func flash(ctx context.Context, devConn dev.DevConn) error {
if err != nil {
return errors.Annotatef(err, "failed to load %s", fwname)
}
if !*keepTempFiles {
if !*flags.KeepTempFiles {
defer fw.Cleanup()
}

Expand Down
145 changes: 145 additions & 0 deletions mos/update/ubuntu.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
//
// Copyright (c) 2014-2019 Cesanta Software Limited
// All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package update

import (
"bufio"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"

"github.com/cesanta/errors"
"github.com/golang/glog"
"github.com/mongoose-os/mos/mos/flags"
"github.com/mongoose-os/mos/mos/ourutil"
"github.com/mongoose-os/mos/mos/version"
)

var (
ubuntuPackageNames = map[UpdateChannel]string{
UpdateChannelRelease: "mos",
UpdateChannelLatest: "mos-latest",
}
ubuntuRepoName = "ppa:mongoose-os/mos"
ubuntuDEBURLPrefix = "https://mongoose-os.com/downloads/debs/"
ubuntuDEBArchiveURL = ubuntuDEBURLPrefix + "index.txt"
ubuntuDEBNameRegexp = regexp.MustCompile(`^(?P<pkg>[^_]+)_(?P<version>[^+]+)\+(?P<hash>[^~]+)\~(?P<distr>[^\d]+)\d+_(?P<arch>[^.]+)\.deb`)
)

func doUbuntuUpdateRepo(oldUpdChannel, newUpdChannel UpdateChannel) error {
oldPkg := ubuntuPackageNames[oldUpdChannel]
newPkg := ubuntuPackageNames[newUpdChannel]

// Start with an apt-get update.
// Do not fail because some unrelated repo may be screwed up
// but our PPA might still be ok.
ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "update")

// Check if mos and mos-latest are among the available packages.
output, err := ourutil.GetCommandOutput("apt-cache", "showpkg", newPkg)
if err != nil {
return errors.Annotatef(err, "faild to get package info")
}
if !strings.Contains(output, "/lists/") {
// No package info in repo lists - we should (re-)add our repo.
// This can happen, for example, after release upgrade which disables 3rd-party repos.
if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-add-repository", "-y", "-u", ubuntuRepoName); err != nil {
return errors.Trace(err)
}
}

if oldPkg != newPkg {
if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "remove", "-y", oldPkg); err != nil {
return errors.Trace(err)
}
}
return ourutil.RunCmd(ourutil.CmdOutAlways, "sudo", "apt-get", "install", "-y", newPkg)
}

func doUbuntuUpdateDEB(oldUpdChannel UpdateChannel, newMosVersion string) error {
distr := version.GetUbuntuBuildIDParts(version.BuildId)["distr"]
arch := runtime.GOARCH
if arch == "arm" {
arch = "armhf"
}
glog.Infof("Distr: %s, arch: %s", distr, arch)
debName := ""
{
glog.Infof("Fetching package archive index from %s...", ubuntuDEBArchiveURL)
resp, err := http.Get(ubuntuDEBArchiveURL)
if err != nil {
return errors.Annotatef(err, "failed to fetch package archive index")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.Errorf("failed to fetch package archive index from %s: %s", ubuntuDEBArchiveURL, resp.Status)
}
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
d := strings.Split(scanner.Text(), " ")[0]
p := ourutil.FindNamedSubmatches(ubuntuDEBNameRegexp, scanner.Text())
glog.V(2).Infof("%s -> %s", scanner.Text(), p)
if p["distr"] == distr && p["version"] == newMosVersion && p["arch"] == arch {
debName = d
break
}
}
if debName == "" {
return errors.Errorf("no package found for %s %s %s", newMosVersion, distr, arch)
}
}
debURL := ubuntuDEBURLPrefix + debName
ourutil.Reportf("Downloading %s...", debURL)
resp, err := http.Get(debURL)
if err != nil {
return errors.Annotatef(err, "failed to fetch package")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.Errorf("failed to fetch package from %s: %s", debURL, resp.Status)
}
debData, err := ioutil.ReadAll(resp.Body)
if err != nil {
return errors.Annotatef(err, "failed to fetch package")
}
tempDir, err := ioutil.TempDir("", "mosdeb-")
if err != nil {
return errors.Annotatef(err, "failed to create staging directory")
}
if !*flags.KeepTempFiles {
defer os.RemoveAll(tempDir)
}
debFile := filepath.Join(tempDir, debName)
if err = ioutil.WriteFile(debFile, debData, 0644); err != nil {
return errors.Annotatef(err, "failed to write package file")
}
glog.Infof("Wrote %s %s", debFile, resp.Status)
ourutil.Reportf("Removing installed package (if any)...")
oldPkg := ubuntuPackageNames[oldUpdChannel]
ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "remove", "-y", oldPkg)
ourutil.Reportf("Installing new package...")
if err = ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "dpkg", "-i", debFile); err != nil {
return errors.Annotatef(err, "failed to install %s", debFile)
}
ourutil.Reportf("Package installed")
return nil
}
41 changes: 5 additions & 36 deletions mos/update/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,6 @@ var (
UpdateChannelRelease: "mos",
UpdateChannelLatest: "mos-latest",
}
ubuntuPackageNames = map[UpdateChannel]string{
UpdateChannelRelease: "mos",
UpdateChannelLatest: "mos-latest",
}
ubuntuRepoName = "ppa:mongoose-os/mos"
)

// mosVersion can be either exact mos version like "1.6", or update channel
Expand Down Expand Up @@ -108,36 +103,6 @@ func GetServerMosVersion(mosVersion string, extraInfo ...string) (*version.Versi
return &serverVersion, nil
}

func doUbuntuUpdate(oldUpdChannel, newUpdChannel UpdateChannel) error {
oldPkg := ubuntuPackageNames[oldUpdChannel]
newPkg := ubuntuPackageNames[newUpdChannel]

// Start with an apt-get update.
// Do not fail because some unrelated repo may be screwed up
// but our PPA might still be ok.
ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "update")

// Check if mos and mos-latest are among the available packages.
output, err := ourutil.GetCommandOutput("apt-cache", "showpkg", newPkg)
if err != nil {
return errors.Annotatef(err, "faild to get package info")
}
if !strings.Contains(output, "/lists/") {
// No package info in repo lists - we should (re-)add our repo.
// This can happen, for example, after release upgrade which disables 3rd-party repos.
if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-add-repository", "-y", "-u", ubuntuRepoName); err != nil {
return errors.Trace(err)
}
}

if oldPkg != newPkg {
if err := ourutil.RunCmd(ourutil.CmdOutOnError, "sudo", "apt-get", "remove", "-y", oldPkg); err != nil {
return errors.Trace(err)
}
}
return ourutil.RunCmd(ourutil.CmdOutAlways, "sudo", "apt-get", "install", "-y", newPkg)
}

func doBrewUpdate(oldUpdChannel, newUpdChannel UpdateChannel) error {
oldPkg := brewPackageNames[oldUpdChannel]
newPkg := brewPackageNames[newUpdChannel]
Expand Down Expand Up @@ -174,7 +139,11 @@ func Update(ctx context.Context, devConn dev.DevConn) error {
}

if version.LooksLikeUbuntuBuildId(version.BuildId) {
return doUbuntuUpdate(updChannel, newUpdChannel)
if newMosVersion == string(newUpdChannel) {
return doUbuntuUpdateRepo(updChannel, newUpdChannel)
} else {
return doUbuntuUpdateDEB(updChannel, newMosVersion)
}
} else if version.LooksLikeBrewBuildId(version.BuildId) {
return doBrewUpdate(updChannel, newUpdChannel)
} else if version.LooksLikeDistrBuildId(version.BuildId) {
Expand Down
27 changes: 15 additions & 12 deletions mos/version/version_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,7 @@ const (

var (
regexpVersionNumber = regexp.MustCompile(`^\d+\.[0-9.]*$`)
regexpBuildIdDistr = regexp.MustCompile(
`^(?P<version>[^+]+)\+(?P<hash>[^~]+)\~(?P<distr>.+)$`,
)
regexpBuildIdDistr = regexp.MustCompile(`^(?P<version>[^+]+)\+(?P<hash>[^~]+)\~(?P<distr>[^\d]+)\d+$`)

ubuntuDistrNames = []string{"xenial", "bionic", "cosmic", "disco"}
)
Expand Down Expand Up @@ -85,21 +83,26 @@ func LooksLikeBrewBuildId(s string) bool {
// debian build id, returns either "latest" or "release". Otherwise, returns
// an empty string.
func GetUbuntuUpdateChannel(buildId string) string {
matches := ourutil.FindNamedSubmatches(regexpBuildIdDistr, buildId)
if matches != nil {
for _, v := range ubuntuDistrNames {
if strings.HasPrefix(matches["distr"], v) {
if LooksLikeVersionNumber(matches["version"]) {
return "release"
} else {
return "latest"
}
parts := GetUbuntuBuildIDParts(buildId)
if parts == nil {
return ""
}
for _, v := range ubuntuDistrNames {
if strings.HasPrefix(parts["distr"], v) {
if LooksLikeVersionNumber(parts["version"]) {
return "release"
} else {
return "latest"
}
}
}
return ""
}

func GetUbuntuBuildIDParts(buildId string) map[string]string {
return ourutil.FindNamedSubmatches(regexpBuildIdDistr, buildId)
}

func GetUserAgent() string {
return fmt.Sprintf("mos/%s %s (%s; %s)", Version, BuildId, runtime.GOOS, runtime.GOARCH)
}

0 comments on commit db6f2c4

Please sign in to comment.