Merge branch 'main' of github.com:threefoldtech/zos into development_…

…rollout_update

.golangci.yml

@@ -0,0 +1,17 @@
+linters:
+  enable:
+    - errcheck
+    - gofmt
+    - govet
+    - ineffassign
+    - unconvert
+    - staticcheck
+    - gocyclo
+  enable-all: false
+linters-settings:
+  gocyclo:
+    # Minimal code complexity to report.
+    # Default: 30 (but we recommend 10-20)
+    min-complexity: 100
+run:
+  timeout: 20m

CODEOWNERS

@@ -1,2 +1 @@
-*   @muhamadazmy @delandtj @MarioBassem @ashraffouda @rawdaGastan @xmonader
-
+*   @muhamadazmy @Omarabdul3ziz @ashraffouda @rawdaGastan @xmonader @Eslam-Nawara

bins/packages/mycelium/mycelium.sh

@@ -1,5 +1,5 @@
-MYCELIUM_VERSION="0.5.3"
-MYCELIUM_CHECKSUM="a41979ca85b2d60ec4757a2dbd88e95c"
+MYCELIUM_VERSION="0.5.4"
+MYCELIUM_CHECKSUM="44dbdde5cda639a16d0999b8889436a3"
 MYCELIUM_LINK="https://github.com/threefoldtech/mycelium/releases/download/v${MYCELIUM_VERSION}/mycelium-x86_64-unknown-linux-musl.tar.gz"
 
 download_mycelium() {

bootstrap/bootstrap/README.md

@@ -17,7 +17,29 @@ will do a multiple stage bootstrap. Currently this is only two stages:
 ## How to works
 
 - Bootstrap is used by [0-initramfs](https://github.com/threefoldtech/0-initramfs/blob/development-zos-v3/packages/modules.sh) to basically add `internet` and `bootstrap` services to the base image
-- After internet service is fully started, bootstrap will start to download flists needed to for zos node to work properly
+- After internet service is fully started, bootstrap will start to download flists needed for zos node to work properly
 - As described above bootstrap run in two stages:
   - The first stage is used to update bootstrap itself, and it is done like that to avoid re-building the image if we only changed the bootstrap code. this update is basically done from `tf-autobuilder` repo in the [hub/tf-autobuilder](https://hub.grid.tf/tf-autobuilder) and download the latest bootstrap flist
   - For the second stage bootstrap will download the flists for that env. bootstrap cares about `runmode` argument that we pass during the start of the node. for example if we passed `runmode=dev` it will get the the tag `development` under [hub/tf-zos](https://hub.grid.tf/tf-zos) each tag is linked to a sub-directory where all flists for this env exists to be downloaded and installed on the node
+
+## Testing in Developer setup
+
+To test bootstrap changes on a local dev-setup you need to do the following
+
+- under zos/qemu `cp -r overlay.normal overlay.custom`
+- build `bootstrap` bin
+- copy the `bootstrap` bin to overlay.custom/sbin/
+- remove dir `overlay.custom/bin`
+- remove all files under `overlay.custom/etc/zinit/`
+- add the file overlay.custom/etc/zinit/bootstrap.yaml with the following content
+
+```
+exec: bootstrap -d
+oneshot: true
+after:
+  - internet
+```
+
+- remove overlay link under `qemu/overlay `
+- create a new link pointing to overlay.custom under zos/qemu `ln -s overlay.custom overlay`
+- boot your vm as normal

bootstrap/bootstrap/src/bootstrap.rs

@@ -8,63 +8,16 @@ use super::workdir::WorkDir;
 use super::zfs::Zfs;
 use super::zinit;
 use anyhow::{Context, Result};
-use config::{RunMode, Version};
+use config::Version;
 use retry;
 
 const ZOS_REPO: &str = "tf-zos";
-const BIN_REPO_V2: &str = "tf-zos-bins";
-const BIN_REPO_V3: &str = "tf-zos-v3-bins";
 
-const FLIST_INFO_FILE: &str = "/tmp/flist.info";
-const FLIST_NAME_FILE: &str = "/tmp/flist.name";
 const FLIST_TAG_FILE: &str = "/tmp/tag.info";
 const BOOTSTAP_FLIST: &str = "bootstrap:latest.flist";
 
 const WORKDIR: &str = "/tmp/bootstrap";
 
-fn bootstrap_zos(cfg: &config::Config) -> Result<()> {
-    let flist = match &cfg.runmode {
-        RunMode::Prod => match &cfg.version {
-            Version::V3 => "zos:production-3:latest.flist",
-        },
-        RunMode::Dev => match &cfg.version {
-            Version::V3 => "zos:development-3:latest.flist",
-        },
-        RunMode::Test => match &cfg.version {
-            Version::V3 => "zos:testing-3:latest.flist",
-        },
-        RunMode::QA => match &cfg.version {
-            Version::V3 => "zos:qa-3:latest.flist",
-        },
-    };
-
-    debug!("using flist: {}/{}", ZOS_REPO, flist);
-    let repo = hub::Repo::new(ZOS_REPO);
-    let flist = retry::retry(retry::delay::Exponential::from_millis(200), || {
-        info!("get flist info: {}", flist);
-        let info = match repo.get(flist) {
-            Ok(info) => info,
-            Err(err) => {
-                error!("failed to get info: {}", err);
-                bail!("failed to get info: {}", err);
-            }
-        };
-
-        Ok(info)
-    });
-
-    let flist = match flist {
-        Ok(flist) => flist,
-        Err(e) => bail!("failed to download flist: {:?}", e),
-    };
-
-    // write down boot info for other system components (like upgraded)
-    flist.write(FLIST_INFO_FILE)?;
-    std::fs::write(FLIST_NAME_FILE, format!("{}/{}", ZOS_REPO, flist.name))?;
-
-    install_package(&flist)
-}
-
 /// update stage make sure we are running latest
 /// version of bootstrap
 pub fn update(_cfg: &config::Config) -> Result<()> {
@@ -116,11 +69,16 @@ pub fn install(cfg: &config::Config) -> Result<()> {
     let repo = Repo::new(ZOS_REPO);
 
     let runmode = cfg.runmode.to_string();
-    // we need to list all taglinks inside the repo
 
+    let mut listname = runmode.clone();
+    match cfg.version {
+        Version::V3 => {}
+        Version::V4 => listname = format!("{}-v4", runmode),
+    }
+    // we need to list all taglinks
     let mut tag = None;
     for list in repo.list()? {
-        if list.kind == Kind::TagLink && list.name == runmode {
+        if list.kind == Kind::TagLink && list.name == listname {
             tag = Some(list);
             break;
         }
@@ -133,12 +91,7 @@ pub fn install(cfg: &config::Config) -> Result<()> {
     let result = WorkDir::run(WORKDIR, || -> Result<()> {
         match tag {
             None => {
-                // old style bootstrap.
-                // we need to install binaries and zos from 2 different
-                // places
-                // we also track which binaries are installed individually
-                install_packages_old(cfg)?;
-                bootstrap_zos(cfg)
+                bail!("no tag found attached to this version")
             }
             Some(tag) => {
                 // new style bootstrap
@@ -160,57 +113,6 @@ pub fn install(cfg: &config::Config) -> Result<()> {
     result
 }
 
-fn install_packages_old(cfg: &config::Config) -> Result<()> {
-    let name = match cfg.version {
-        Version::V3 => BIN_REPO_V3,
-    };
-
-    let repo = match cfg.runmode {
-        config::RunMode::Prod => name.into(),
-        config::RunMode::Dev => format!("{}.dev", name),
-        config::RunMode::Test => format!("{}.test", name),
-        config::RunMode::QA => format!("{}.qanet", name),
-    };
-
-    let client = hub::Repo::new(&repo);
-    let packages = retry::retry(retry::delay::Exponential::from_millis(200), || {
-        info!("list packages in: {}", BIN_REPO_V2);
-        //the full point of this match is the logging.
-        let packages = match client.list() {
-            Ok(info) => info,
-            Err(err) => {
-                error!("failed to list repo '{}': {}", BIN_REPO_V2, err);
-                bail!("failed to list repo '{}': {}", BIN_REPO_V2, err);
-            }
-        };
-
-        Ok(packages)
-    });
-
-    let packages = match packages {
-        Ok(packages) => packages,
-        Err(err) => bail!("failed to list '{}': {:?}", BIN_REPO_V2, err),
-    };
-
-    let mut map = std::collections::HashMap::new();
-    for package in packages.iter() {
-        match install_package(package) {
-            Ok(_) => {}
-            Err(err) => warn!("failed to install package '{}': {}", package.url, err),
-        };
-
-        map.insert(format!("{}/{}", repo, package.name), package.clone());
-    }
-
-    let output = std::fs::OpenOptions::new()
-        .create(true)
-        .write(true)
-        .open("/tmp/bins.info")?;
-    serde_json::to_writer(&output, &map)?;
-
-    Ok(())
-}
-
 fn install_packages(packages: &[Flist]) -> Result<()> {
     for package in packages {
         install_package(&package)?;