diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 000000000..e90d26d05 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,17 @@ +linters: + enable: + - errcheck + - gofmt + - govet + - ineffassign + - unconvert + - staticcheck + - gocyclo + enable-all: false +linters-settings: + gocyclo: + # Minimal code complexity to report. + # Default: 30 (but we recommend 10-20) + min-complexity: 100 +run: + timeout: 20m diff --git a/CODEOWNERS b/CODEOWNERS index 453aeaddb..40861d5f6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1 @@ -* @muhamadazmy @delandtj @MarioBassem @ashraffouda @rawdaGastan @xmonader - +* @muhamadazmy @Omarabdul3ziz @ashraffouda @rawdaGastan @xmonader @Eslam-Nawara diff --git a/bins/packages/mycelium/mycelium.sh b/bins/packages/mycelium/mycelium.sh index e988c2b1d..24d626ac5 100644 --- a/bins/packages/mycelium/mycelium.sh +++ b/bins/packages/mycelium/mycelium.sh @@ -1,5 +1,5 @@ -MYCELIUM_VERSION="0.5.3" -MYCELIUM_CHECKSUM="a41979ca85b2d60ec4757a2dbd88e95c" +MYCELIUM_VERSION="0.5.4" +MYCELIUM_CHECKSUM="44dbdde5cda639a16d0999b8889436a3" MYCELIUM_LINK="https://github.com/threefoldtech/mycelium/releases/download/v${MYCELIUM_VERSION}/mycelium-x86_64-unknown-linux-musl.tar.gz" download_mycelium() { diff --git a/bootstrap/bootstrap/Cargo.lock b/bootstrap/bootstrap/Cargo.lock index 8f3f5b470..d8f21893d 100644 --- a/bootstrap/bootstrap/Cargo.lock +++ b/bootstrap/bootstrap/Cargo.lock @@ -155,6 +155,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "encoding_rs" version = "0.8.32" @@ -528,6 +537,12 @@ dependencies = [ "void", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num_cpus" version = "1.15.0" @@ -576,7 +591,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -631,20 +646,26 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -877,22 +898,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "5b3e4cd94123dd520a128bcd11e34d9e9e423e7e3e50425cb1b4b1e3549d0284" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.206" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "fabfb6138d2383ea8208cf98ccf69cdfb1aff4088460681d84189aa259762f97" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.74", ] [[package]] @@ -973,6 +994,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -998,13 +1030,16 @@ dependencies = [ [[package]] name = "time" -version = "0.3.19" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa", "libc", + "num-conv", "num_threads", + "powerfmt", "serde", "time-core", "time-macros", @@ -1012,16 +1047,17 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.7" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -1217,7 +1253,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-shared", ] @@ -1251,7 +1287,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/bootstrap/bootstrap/README.md b/bootstrap/bootstrap/README.md index 5374ba3a3..e36b4bfd3 100644 --- a/bootstrap/bootstrap/README.md +++ b/bootstrap/bootstrap/README.md @@ -17,7 +17,29 @@ will do a multiple stage bootstrap. Currently this is only two stages: ## How to works - Bootstrap is used by [0-initramfs](https://github.com/threefoldtech/0-initramfs/blob/development-zos-v3/packages/modules.sh) to basically add `internet` and `bootstrap` services to the base image -- After internet service is fully started, bootstrap will start to download flists needed to for zos node to work properly +- After internet service is fully started, bootstrap will start to download flists needed for zos node to work properly - As described above bootstrap run in two stages: - The first stage is used to update bootstrap itself, and it is done like that to avoid re-building the image if we only changed the bootstrap code. this update is basically done from `tf-autobuilder` repo in the [hub/tf-autobuilder](https://hub.grid.tf/tf-autobuilder) and download the latest bootstrap flist - For the second stage bootstrap will download the flists for that env. bootstrap cares about `runmode` argument that we pass during the start of the node. for example if we passed `runmode=dev` it will get the the tag `development` under [hub/tf-zos](https://hub.grid.tf/tf-zos) each tag is linked to a sub-directory where all flists for this env exists to be downloaded and installed on the node + +## Testing in Developer setup + +To test bootstrap changes on a local dev-setup you need to do the following + +- under zos/qemu `cp -r overlay.normal overlay.custom` +- build `bootstrap` bin +- copy the `bootstrap` bin to overlay.custom/sbin/ +- remove dir `overlay.custom/bin` +- remove all files under `overlay.custom/etc/zinit/` +- add the file overlay.custom/etc/zinit/bootstrap.yaml with the following content + +``` +exec: bootstrap -d +oneshot: true +after: + - internet +``` + +- remove overlay link under `qemu/overlay ` +- create a new link pointing to overlay.custom under zos/qemu `ln -s overlay.custom overlay` +- boot your vm as normal diff --git a/bootstrap/bootstrap/src/bootstrap.rs b/bootstrap/bootstrap/src/bootstrap.rs index 5e1c85ec8..76498c1d7 100644 --- a/bootstrap/bootstrap/src/bootstrap.rs +++ b/bootstrap/bootstrap/src/bootstrap.rs @@ -8,63 +8,16 @@ use super::workdir::WorkDir; use super::zfs::Zfs; use super::zinit; use anyhow::{Context, Result}; -use config::{RunMode, Version}; +use config::Version; use retry; const ZOS_REPO: &str = "tf-zos"; -const BIN_REPO_V2: &str = "tf-zos-bins"; -const BIN_REPO_V3: &str = "tf-zos-v3-bins"; -const FLIST_INFO_FILE: &str = "/tmp/flist.info"; -const FLIST_NAME_FILE: &str = "/tmp/flist.name"; const FLIST_TAG_FILE: &str = "/tmp/tag.info"; const BOOTSTAP_FLIST: &str = "bootstrap:latest.flist"; const WORKDIR: &str = "/tmp/bootstrap"; -fn bootstrap_zos(cfg: &config::Config) -> Result<()> { - let flist = match &cfg.runmode { - RunMode::Prod => match &cfg.version { - Version::V3 => "zos:production-3:latest.flist", - }, - RunMode::Dev => match &cfg.version { - Version::V3 => "zos:development-3:latest.flist", - }, - RunMode::Test => match &cfg.version { - Version::V3 => "zos:testing-3:latest.flist", - }, - RunMode::QA => match &cfg.version { - Version::V3 => "zos:qa-3:latest.flist", - }, - }; - - debug!("using flist: {}/{}", ZOS_REPO, flist); - let repo = hub::Repo::new(ZOS_REPO); - let flist = retry::retry(retry::delay::Exponential::from_millis(200), || { - info!("get flist info: {}", flist); - let info = match repo.get(flist) { - Ok(info) => info, - Err(err) => { - error!("failed to get info: {}", err); - bail!("failed to get info: {}", err); - } - }; - - Ok(info) - }); - - let flist = match flist { - Ok(flist) => flist, - Err(e) => bail!("failed to download flist: {:?}", e), - }; - - // write down boot info for other system components (like upgraded) - flist.write(FLIST_INFO_FILE)?; - std::fs::write(FLIST_NAME_FILE, format!("{}/{}", ZOS_REPO, flist.name))?; - - install_package(&flist) -} - /// update stage make sure we are running latest /// version of bootstrap pub fn update(_cfg: &config::Config) -> Result<()> { @@ -116,11 +69,16 @@ pub fn install(cfg: &config::Config) -> Result<()> { let repo = Repo::new(ZOS_REPO); let runmode = cfg.runmode.to_string(); - // we need to list all taglinks inside the repo + let mut listname = runmode.clone(); + match cfg.version { + Version::V3 => {} + Version::V4 => listname = format!("{}-v4", runmode), + } + // we need to list all taglinks let mut tag = None; for list in repo.list()? { - if list.kind == Kind::TagLink && list.name == runmode { + if list.kind == Kind::TagLink && list.name == listname { tag = Some(list); break; } @@ -133,12 +91,7 @@ pub fn install(cfg: &config::Config) -> Result<()> { let result = WorkDir::run(WORKDIR, || -> Result<()> { match tag { None => { - // old style bootstrap. - // we need to install binaries and zos from 2 different - // places - // we also track which binaries are installed individually - install_packages_old(cfg)?; - bootstrap_zos(cfg) + bail!("no tag found attached to this version") } Some(tag) => { // new style bootstrap @@ -160,57 +113,6 @@ pub fn install(cfg: &config::Config) -> Result<()> { result } -fn install_packages_old(cfg: &config::Config) -> Result<()> { - let name = match cfg.version { - Version::V3 => BIN_REPO_V3, - }; - - let repo = match cfg.runmode { - config::RunMode::Prod => name.into(), - config::RunMode::Dev => format!("{}.dev", name), - config::RunMode::Test => format!("{}.test", name), - config::RunMode::QA => format!("{}.qanet", name), - }; - - let client = hub::Repo::new(&repo); - let packages = retry::retry(retry::delay::Exponential::from_millis(200), || { - info!("list packages in: {}", BIN_REPO_V2); - //the full point of this match is the logging. - let packages = match client.list() { - Ok(info) => info, - Err(err) => { - error!("failed to list repo '{}': {}", BIN_REPO_V2, err); - bail!("failed to list repo '{}': {}", BIN_REPO_V2, err); - } - }; - - Ok(packages) - }); - - let packages = match packages { - Ok(packages) => packages, - Err(err) => bail!("failed to list '{}': {:?}", BIN_REPO_V2, err), - }; - - let mut map = std::collections::HashMap::new(); - for package in packages.iter() { - match install_package(package) { - Ok(_) => {} - Err(err) => warn!("failed to install package '{}': {}", package.url, err), - }; - - map.insert(format!("{}/{}", repo, package.name), package.clone()); - } - - let output = std::fs::OpenOptions::new() - .create(true) - .write(true) - .open("/tmp/bins.info")?; - serde_json::to_writer(&output, &map)?; - - Ok(()) -} - fn install_packages(packages: &[Flist]) -> Result<()> { for package in packages { install_package(&package)?; diff --git a/bootstrap/bootstrap/src/config.rs b/bootstrap/bootstrap/src/config.rs index 1a918fc23..f6a5da5eb 100644 --- a/bootstrap/bootstrap/src/config.rs +++ b/bootstrap/bootstrap/src/config.rs @@ -28,6 +28,7 @@ impl Display for RunMode { #[derive(Debug)] pub enum Version { V3, + V4, } fn runmode() -> Result { @@ -64,6 +65,7 @@ fn version() -> Result { Some(input) => match input { Some(input) => match input.as_ref() { "v3" => Version::V3, + "v4" => Version::V4, m => { bail!("unknown version: {}", m); } @@ -117,12 +119,15 @@ impl Config { if stage == 0 { bail!("invalid stage value 0, stages starting from 1"); } - + let mut ver = Version::V3; + if stage == 2 { + ver = version()? + } Ok(Config { - stage: stage, + stage, debug: matches.occurrences_of("debug") > 0, runmode: runmode()?, - version: version()?, + version: ver, }) } } diff --git a/bootstrap/bootstrap/src/kparams.rs b/bootstrap/bootstrap/src/kparams.rs index d960adbd6..61e7f54be 100644 --- a/bootstrap/bootstrap/src/kparams.rs +++ b/bootstrap/bootstrap/src/kparams.rs @@ -40,10 +40,11 @@ mod tests { #[test] fn test_parse() -> Result<(), Error> { - let input: &str = "initrd=initramfs-linux.img root=UUID=10f9e7bb-ba63-4fbd-a95e-c78b5496cfbe rootflags=subvol=root rw b43.allhwsupport=1"; + let input: &str = "initrd=initramfs-linux.img version=v3 root=UUID=10f9e7bb-ba63-4fbd-a95e-c78b5496cfbe rootflags=subvol=root rw b43.allhwsupport=1"; let result = parse(input.as_bytes())?; - assert_eq!(result.len(), 5); + assert_eq!(result.len(), 6); assert_eq!(result["rw"], None); + assert_eq!(result["version"], Some(String::from("v3"))); assert_eq!(result["rootflags"], Some(String::from("subvol=root"))); Ok(()) } diff --git a/client/node.go b/client/node.go index 073d0154b..455f448fb 100644 --- a/client/node.go +++ b/client/node.go @@ -212,7 +212,6 @@ func (n *NodeClient) Counters(ctx context.Context) (counters Counters, err error const cmd = "zos.statistics.get" err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &counters) return - } // Pools returns statistics of separate pools @@ -267,7 +266,6 @@ func (n *NodeClient) NetworkListAllInterfaces(ctx context.Context) (result map[s err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &result) return - } // NetworkSetPublicExitDevice select which physical interface to use as an exit device @@ -322,6 +320,13 @@ func (n *NodeClient) NetworkGetPublicConfig(ctx context.Context) (cfg pkg.Public return } +func (n *NodeClient) SystemGetNodeFeatures(ctx context.Context) (feat []pkg.NodeFeature, err error) { + const cmd = "zos.system.node_features_get" + + err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &feat) + return +} + func (n *NodeClient) SystemVersion(ctx context.Context) (ver Version, err error) { const cmd = "zos.system.version" diff --git a/cmds/modules/powerd/main.go b/cmds/modules/powerd/main.go index 336129767..71a8c3ff3 100644 --- a/cmds/modules/powerd/main.go +++ b/cmds/modules/powerd/main.go @@ -37,6 +37,7 @@ var Module cli.Command = cli.Command{ func action(cli *cli.Context) error { var ( msgBrokerCon string = cli.String("broker") + powerdLabel string = "powerd" ) ctx, _ := utils.WithSignal(cli.Context) @@ -51,16 +52,28 @@ func action(cli *cli.Context) error { return errors.Wrap(err, "failed to connect to message broker server") } + zui := stubs.NewZUIStub(cl) + // empty out zui errors for powerd + if zuiErr := zui.PushErrors(cli.Context, powerdLabel, []string{}); zuiErr != nil { + log.Info().Err(zuiErr).Send() + } + identity := stubs.NewIdentityManagerStub(cl) register := stubs.NewRegistrarStub(cl) nodeID, err := register.NodeID(ctx) if err != nil { + if zuiErr := zui.PushErrors(cli.Context, powerdLabel, []string{err.Error()}); zuiErr != nil { + log.Info().Err(zuiErr).Send() + } return errors.Wrap(err, "failed to get node id") } twinID, err := register.TwinID(ctx) if err != nil { + if zuiErr := zui.PushErrors(cli.Context, powerdLabel, []string{err.Error()}); zuiErr != nil { + log.Info().Err(zuiErr).Send() + } return errors.Wrap(err, "failed to get twin id") } diff --git a/cmds/modules/zui/header.go b/cmds/modules/zui/header.go index 84ccc88a4..8d0fb2c1b 100644 --- a/cmds/modules/zui/header.go +++ b/cmds/modules/zui/header.go @@ -9,6 +9,8 @@ import ( "github.com/gizak/termui/v3/widgets" "github.com/pkg/errors" + + "github.com/rs/zerolog/log" "github.com/threefoldtech/zbus" "github.com/threefoldtech/zos/pkg/app" "github.com/threefoldtech/zos/pkg/environment" @@ -54,6 +56,14 @@ func headerRenderer(ctx context.Context, c zbus.Client, h *widgets.Paragraph, r } go func() { + registrarLable := "registrar" + zui := stubs.NewZUIStub(c) + + // empty out zui errors for registrar + if zuiErr := zui.PushErrors(ctx, registrarLable, []string{}); zuiErr != nil { + log.Info().Err(zuiErr).Send() + } + farmID, _ := identity.FarmID(ctx) for version := range ch { var name string @@ -69,7 +79,10 @@ func headerRenderer(ctx context.Context, c zbus.Client, h *widgets.Paragraph, r if isInProgressError(err) { nodeID = green(err.Error()) } else { - nodeID = red(err.Error()) + nodeID = red(fmt.Sprintf("%d (unregistered)", node)) + if zuiErr := zui.PushErrors(ctx, registrarLable, []string{err.Error()}); zuiErr != nil { + log.Info().Err(zuiErr).Send() + } } } else { nodeID = green(fmt.Sprint(node)) @@ -77,7 +90,7 @@ func headerRenderer(ctx context.Context, c zbus.Client, h *widgets.Paragraph, r cache := green("OK") if app.CheckFlag(app.LimitedCache) { - cache = red("no ssd disks detected") + cache = red("no ssd disks detected, running on hdd-only mode") } else if app.CheckFlag(app.ReadonlyCache) { cache = red("cache is read-only") } diff --git a/docs/manual/api.md b/docs/manual/api.md index 4127a5e4f..250eb00f2 100644 --- a/docs/manual/api.md +++ b/docs/manual/api.md @@ -217,6 +217,22 @@ name must be one of (free) names returned by `zos.network.admin.interfaces` |---|---|---| | `zos.system.hypervisor` | - | `string` | +### Get Node Features + +| command |body| return| +|---|---|---| +| `zos.system.node_features_get` | - |`[]NodeFeature` | + + +Where + +```json +NodeFeature: "string" + +``` + +returns the types of workloads can be deployed depending on the network manager running on the node + ## GPUs ### List Gpus diff --git a/go.mod b/go.mod index b9a2910c7..f09ef6326 100644 --- a/go.mod +++ b/go.mod @@ -39,15 +39,16 @@ require ( github.com/joncrlsn/dque v0.0.0-20200702023911-3e80e3146ce5 github.com/lestrrat-go/jwx v1.1.7 github.com/machinebox/graphql v0.2.2 + github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/rs/zerolog v1.32.0 + github.com/rs/zerolog v1.33.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/stretchr/testify v1.9.0 github.com/threefoldtech/0-fs v1.3.1-0.20240424140157-b488dfedcc56 - github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240416134707-eb36aa90df2d - github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.15.2 + github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240827163226-d4e15e206974 + github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.15.14 github.com/threefoldtech/zbus v1.0.1 github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa @@ -55,8 +56,8 @@ require ( github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f github.com/whs/nacl-sealed-box v0.0.0-20180930164530-92b9ba845d8d github.com/yggdrasil-network/yggdrasil-go v0.4.0 - golang.org/x/crypto v0.23.0 - golang.org/x/sys v0.20.0 + golang.org/x/crypto v0.26.0 + golang.org/x/sys v0.23.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20200609130330-bd2cb7843e1b gopkg.in/yaml.v2 v2.4.0 gotest.tools v2.2.0+incompatible @@ -74,7 +75,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ethereum/go-ethereum v1.11.6 // indirect @@ -89,7 +90,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/gorilla/websocket v1.5.1 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/gtank/ristretto255 v0.1.2 // indirect github.com/hanwen/go-fuse/v2 v2.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -115,7 +116,6 @@ require ( github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nsf/termbox-go v1.1.1 // indirect - github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect github.com/onsi/ginkgo v1.16.4 // indirect github.com/onsi/gomega v1.16.0 // indirect github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect @@ -147,8 +147,8 @@ require ( go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.zx2c4.com/wireguard v0.0.20200320 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 2d5e7ab3c..0174ad46e 100644 --- a/go.sum +++ b/go.sum @@ -124,8 +124,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5il github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 h1:sgNeV1VRMDzs6rzyPpxyM0jp317hnwiq58Filgag2xw= github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0/go.mod h1:J70FGZSbzsjecRTiTzER+3f1KZLNaXkuv+yeFTKoxM8= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/diskfs/go-diskfs v1.2.0 h1:Ow4xorEDw1VNYKbC+SA/qQNwi5gWIwdKUxmUcLFST24= @@ -268,8 +268,8 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v1.5.0 h1:fDqblo50TEpD0LY7RXk/LFVYEVqo3+tXMNMPSVXA1yc= github.com/graph-gophers/graphql-go v1.5.0/go.mod h1:YtmJZDLbF1YYNrlNAuiO5zAStUWc3XZT07iGsVqe1Os= github.com/graph-gophers/graphql-transport-ws v0.0.2 h1:DbmSkbIGzj8SvHei6n8Mh9eLQin8PtA8xY9eCzjRpvo= @@ -510,8 +510,8 @@ github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg= -github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= -github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -556,10 +556,10 @@ github.com/stripe/safesql v0.2.0/go.mod h1:q7b2n0JmzM1mVGfcYpanfVb2j23cXZeWFxcIL github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/threefoldtech/0-fs v1.3.1-0.20240424140157-b488dfedcc56 h1:uWd8JfE8N3IM6Zw/LMr0+sRJl+G0YgqQmNDP8xXR0vw= github.com/threefoldtech/0-fs v1.3.1-0.20240424140157-b488dfedcc56/go.mod h1:lZjR32SiNo3dP70inVFxaLMyZjmKX1ucS+5O31dbPNM= -github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240416134707-eb36aa90df2d h1:vWHR33VfphOahBlo3aGCE/eKPlfb4BCInhKidGkdp5s= -github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240416134707-eb36aa90df2d/go.mod h1:dtDKAPiUDxAwIkfHV7xcAFZcOm+xwNIuOI1MLFS+MeQ= -github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.15.2 h1:6NsugITjq90ErPqa7+JGTYpaduh+eInyIxfy+eBvbIs= -github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.15.2/go.mod h1:/Jf4SDtYO2cov3ZZQhyQqh+4o8UYYUrs+NDHWe/xUwY= +github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240827163226-d4e15e206974 h1:95Ox9WYNiwm3mVJeVF/lp7iCpVGA28AkU1atN893eDo= +github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240827163226-d4e15e206974/go.mod h1:dtDKAPiUDxAwIkfHV7xcAFZcOm+xwNIuOI1MLFS+MeQ= +github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.15.14 h1:tL3A+vQsWWTDBhtl0wfMg1YR/VoK5Dq7xjH/W/vnEso= +github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.15.14/go.mod h1:cDTivSsUl4r1jN+H8/W6yDqByOOliY8HEGuVuL3ixV8= github.com/threefoldtech/zbus v1.0.1 h1:3KaEpyOiDYAw+lrAyoQUGIvY9BcjVRXlQ1beBRqhRNk= github.com/threefoldtech/zbus v1.0.1/go.mod h1:E/v/xEvG/l6z/Oj0aDkuSUXFm/1RVJkhKBwDTAIdsHo= github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0= @@ -637,8 +637,8 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= @@ -653,8 +653,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -701,8 +701,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -762,8 +762,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -772,8 +772,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7-0.20210503195748-5c7c50ebbd4f/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -795,8 +795,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/Makefile b/pkg/Makefile index 2af25b9a0..3b78e60ef 100644 --- a/pkg/Makefile +++ b/pkg/Makefile @@ -3,51 +3,21 @@ GOPATH := $(shell go env GOPATH) all: build -getdeps: - @echo "Installing golint" && go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.0 - @echo "Installing gocyclo" && go install github.com/fzipp/gocyclo/cmd/gocyclo@latest - @echo "Installing deadcode" && go install github.com/remyoudompheng/go-misc/deadcode@latest - @echo "Installing misspell" && go install github.com/client9/misspell/cmd/misspell@latest - @echo "Installing ineffassign" && go install github.com/gordonklaus/ineffassign@latest - @echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest - -verifiers: vet fmt lint cyclo spelling static #deadcode - -vet: - @echo "Running $@" - @go vet -atomic -bool -copylocks -nilfunc -printf -rangeloops -unreachable -unsafeptr -unusedresult ./... +all: getdeps test -fmt: - @echo "Running $@" - @gofmt -d . +getdeps: + @echo "Installing golangci-lint" && go get github.com/golangci/golangci-lint/cmd/golangci-lint && go install github.com/golangci/golangci-lint/cmd/golangci-lint + go mod tidy lint: @echo "Running $@" - @${GOPATH}/bin/golangci-lint run - -ineffassign: - @echo "Running $@" - @${GOPATH}/bin/ineffassign . - -cyclo: - @echo "Running $@" - @${GOPATH}/bin/gocyclo -over 100 . - -deadcode: - @echo "Running $@" - @${GOPATH}/bin/deadcode -test $(shell go list ./...) || true - -spelling: - @${GOPATH}/bin/misspell -i "monitord,forumla,etherent" -error `find .` - -static: - @${GOPATH}/bin/staticcheck -- ./... + @${GOPATH}/bin/golangci-lint run -c ../.golangci.yml check: test -test: verifiers build +test: lint build go test -vet=off -v $(shell go list ./... | grep -Ev "stubs|network" ) -testrace: verifiers build +testrace: lint build go test -vet=off -v $(shell go list ./... | grep -Ev "stubs|network" ) generate: diff --git a/pkg/api_gateway.go b/pkg/api_gateway.go index ba6f6b1c1..58ce6df10 100644 --- a/pkg/api_gateway.go +++ b/pkg/api_gateway.go @@ -10,7 +10,7 @@ import ( type SubstrateGateway interface { CreateNode(node substrate.Node) (uint32, error) CreateTwin(relay string, pk []byte) (uint32, error) - EnsureAccount(activationURL string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) + EnsureAccount(activationURL []string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) GetContract(id uint64) (substrate.Contract, SubstrateError) GetContractIDByNameRegistration(name string) (uint64, SubstrateError) GetFarm(id uint32) (substrate.Farm, error) diff --git a/pkg/container/container.go b/pkg/container/container.go index 10130d7f6..8d7e547bd 100644 --- a/pkg/container/container.go +++ b/pkg/container/container.go @@ -426,7 +426,7 @@ func (c *Module) start(ns, id string) error { ctx := namespaces.WithNamespace(context.Background(), ns) - container, err := client.LoadContainer(ctx, string(id)) + container, err := client.LoadContainer(ctx, id) if err != nil { return err } @@ -615,7 +615,7 @@ func (c *Module) Delete(ns string, id pkg.ContainerID) error { _ = task.Kill(ctx, syscall.SIGTERM) select { case <-exitC: - case <-time.After(time.Duration(shutdownTimeout)): + case <-time.After(shutdownTimeout): log.Debug().Str("id", string(id)).Int("signal", int(syscall.SIGKILL)).Msg("sending signal") _ = task.Kill(ctx, syscall.SIGKILL) select { diff --git a/pkg/container/stats/stats.go b/pkg/container/stats/stats.go index 967e02f70..90e3e802d 100644 --- a/pkg/container/stats/stats.go +++ b/pkg/container/stats/stats.go @@ -46,7 +46,7 @@ func Monitor(addr string, ns string, id string, backend io.WriteCloser) error { ctx := namespaces.WithNamespace(context.Background(), ns) - container, err := client.LoadContainer(ctx, string(id)) + container, err := client.LoadContainer(ctx, id) if err != nil { log.Error().Err(err).Msg("metric container") return err diff --git a/pkg/crypto/crypto_test.go b/pkg/crypto/crypto_test.go index c6ac7c8d2..c0c21714a 100644 --- a/pkg/crypto/crypto_test.go +++ b/pkg/crypto/crypto_test.go @@ -77,7 +77,8 @@ func TestPyNACLCompatibilityEncryption(t *testing.T) { sk := ed25519.NewKeyFromSeed(seed) chiper := "" - fmt.Sscanf("0bfe9e3b9ce17fe6d570b165ea2a01034326b8c81d5f2c5384c8fe886552f074ec43017465598c4f5a857b495b445be46c3df48d14878bd0b1b907", "%x", &chiper) + _, err := fmt.Sscanf("0bfe9e3b9ce17fe6d570b165ea2a01034326b8c81d5f2c5384c8fe886552f074ec43017465598c4f5a857b495b445be46c3df48d14878bd0b1b907", "%x", &chiper) + require.NoError(t, err) decrypted, err := Decrypt([]byte(chiper), sk) require.NoError(t, err) diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 6888565aa..50fca3f6f 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -48,8 +48,8 @@ type Environment struct { // SINCE RELAYS FOR A NODE IS STORED ON THE CHAIN IN A LIMITED SPACE // PLEASE MAKE SURE THAT ANY ENV HAS NO MORE THAN FOUR RELAYS CONFIGURED RelayURL []string - ActivationURL string - GraphQL string + ActivationURL []string + GraphQL []string // private vlan to join // if set, zos will use this as its priv vlan @@ -108,30 +108,44 @@ var ( RunningMode: RunningDev, SubstrateURL: []string{ "wss://tfchain.dev.grid.tf/", + "wss://tfchain.02.dev.grid.tf", }, RelayURL: []string{ "wss://relay.dev.grid.tf", "wss://relay.02.dev.grid.tf", }, - ActivationURL: "https://activation.dev.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins.dev", - GraphQL: "https://graphql.dev.grid.tf/graphql", + ActivationURL: []string{ + "https://activation.dev.grid.tf/activation/activate", + "https://activation.02.dev.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins.dev", + GraphQL: []string{ + "https://graphql.dev.grid.tf/graphql", + "https://graphql.02.dev.grid.tf/graphql", + }, } envTest = Environment{ RunningMode: RunningTest, SubstrateURL: []string{ "wss://tfchain.test.grid.tf/", + "wss://tfchain.02.test.grid.tf", }, RelayURL: []string{ "wss://relay.test.grid.tf", "wss://relay.02.test.grid.tf", }, - ActivationURL: "https://activation.test.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins.test", - GraphQL: "https://graphql.test.grid.tf/graphql", + ActivationURL: []string{ + "https://activation.test.grid.tf/activation/activate", + "https://activation.02.test.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins.test", + GraphQL: []string{ + "https://graphql.test.grid.tf/graphql", + "https://graphql.02.test.grid.tf/graphql", + }, } envQA = Environment{ @@ -142,17 +156,25 @@ var ( }, RelayURL: []string{ "wss://relay.qa.grid.tf", + "wss://relay.02.qa.grid.tf", + }, + ActivationURL: []string{ + "https://activation.qa.grid.tf/activation/activate", + "https://activation.02.qa.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins.qanet", + GraphQL: []string{ + "https://graphql.qa.grid.tf/graphql", + "https://graphql.02.qa.grid.tf/graphql", }, - ActivationURL: "https://activation.qa.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins.qanet", - GraphQL: "https://graphql.qa.grid.tf/graphql", } envProd = Environment{ RunningMode: RunningMain, SubstrateURL: []string{ "wss://tfchain.grid.tf/", + "wss://tfchain.02.grid.tf", "wss://02.tfchain.grid.tf/", "wss://03.tfchain.grid.tf/", "wss://04.tfchain.grid.tf/", @@ -161,10 +183,16 @@ var ( "wss://relay.grid.tf", "wss://relay.02.grid.tf", }, - ActivationURL: "https://activation.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins", - GraphQL: "https://graphql.grid.tf/graphql", + ActivationURL: []string{ + "https://activation.grid.tf/activation/activate", + "https://activation.02.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins", + GraphQL: []string{ + "https://graphql.grid.tf/graphql", + "https://graphql.02.grid.tf/graphql", + }, } ) @@ -241,7 +269,7 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) { if activation, ok := params.Get("activation"); ok { if len(activation) > 0 { - env.ActivationURL = activation[len(activation)-1] + env.ActivationURL = activation } } diff --git a/pkg/flist/cleanup.go b/pkg/flist/cleanup.go index 1dc078d21..0c24ef8b4 100644 --- a/pkg/flist/cleanup.go +++ b/pkg/flist/cleanup.go @@ -59,7 +59,7 @@ func (f *flistModule) cleanCache(now time.Time, age time.Duration) error { if sys, ok := sys.(*syscall.Stat_t); ok { // int64 cast required for arm32 targets - atime := time.Unix(int64(sys.Atim.Sec), int64(sys.Atim.Nsec)) + atime := time.Unix(sys.Atim.Sec, sys.Atim.Nsec) if now.Sub(atime) > age { if err := os.Remove(path); err != nil { diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index 33136241b..b4a4aff53 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -532,7 +532,7 @@ func (g *gatewayModule) configPath(name string) string { func (g *gatewayModule) validateNameContract(name string, twinID uint32) error { - contractID, subErr := g.substrateGateway.GetContractIDByNameRegistration(context.Background(), string(name)) + contractID, subErr := g.substrateGateway.GetContractIDByNameRegistration(context.Background(), name) if subErr.IsCode(pkg.CodeNotFound) { return ErrContractNotReserved } @@ -633,7 +633,7 @@ func (g *gatewayModule) setupRouting(ctx context.Context, wlID string, fqdn stri backend := config.Backends[0] - if err := zos.Backend(backend).Valid(config.TLSPassthrough); err != nil { + if err := backend.Valid(config.TLSPassthrough); err != nil { return errors.Wrapf(err, "failed to validate backend '%s'", backend) } diff --git a/pkg/gridtypes/id.go b/pkg/gridtypes/id.go index 944deec18..40c43db3e 100644 --- a/pkg/gridtypes/id.go +++ b/pkg/gridtypes/id.go @@ -71,7 +71,7 @@ func (i WorkloadID) Unique(n string) string { b = b[:13] } - return string(b) + return b } // IsValidName validates workload name @@ -80,6 +80,10 @@ func IsValidName(n Name) error { return fmt.Errorf("name cannot be empty") } + if len(n) > 50 { + return fmt.Errorf("name cannot exceed 50 characters") + } + if !nameMatch.MatchString(string(n)) { return fmt.Errorf("unsupported character in workload name") } diff --git a/pkg/gridtypes/workload_test.go b/pkg/gridtypes/workload_test.go index cf0f8d8c6..645fea3af 100644 --- a/pkg/gridtypes/workload_test.go +++ b/pkg/gridtypes/workload_test.go @@ -21,7 +21,7 @@ func TestTimestamp(t *testing.T) { exp, err := json.Marshal(n) require.NoError(err) - err = json.Unmarshal([]byte(exp), &v) + err = json.Unmarshal(exp, &v) require.NoError(err) require.Equal(Timestamp(n.Unix()), v) diff --git a/pkg/gridtypes/zos/network.go b/pkg/gridtypes/zos/network.go index 854cfbc41..972af84ea 100644 --- a/pkg/gridtypes/zos/network.go +++ b/pkg/gridtypes/zos/network.go @@ -32,7 +32,7 @@ func NetworkID(twin uint32, network gridtypes.Name) NetID { if len(b) > 13 { b = b[:13] } - return NetID(string(b)) + return NetID(b) } func NetworkIDFromWorkloadID(wl gridtypes.WorkloadID) (NetID, error) { diff --git a/pkg/monitor.go b/pkg/monitor.go index 6e3f30199..39a6de7da 100644 --- a/pkg/monitor.go +++ b/pkg/monitor.go @@ -78,6 +78,9 @@ type PoolStats struct { // PoolsStats alias for map[string]PoolStats type PoolsStats map[string]PoolStats +// Types of workloads can be deployed depending on the network manager running on the node +type NodeFeature string + // SystemMonitor interface (provided by noded) type SystemMonitor interface { NodeID() uint32 @@ -85,6 +88,7 @@ type SystemMonitor interface { CPU(ctx context.Context) <-chan TimesStat Disks(ctx context.Context) <-chan DisksIOCountersStat Nics(ctx context.Context) <-chan NicsIOCounterStat + GetNodeFeatures() []NodeFeature } // HostMonitor interface (provided by noded) diff --git a/pkg/monitord/system.go b/pkg/monitord/system.go index 0f3853a75..05ac7f35c 100644 --- a/pkg/monitord/system.go +++ b/pkg/monitord/system.go @@ -10,11 +10,10 @@ import ( "github.com/shirou/gopsutil/mem" "github.com/shirou/gopsutil/net" "github.com/threefoldtech/zos/pkg" + "github.com/threefoldtech/zos/pkg/gridtypes/zos" ) -var ( - _ pkg.SystemMonitor = (*systemMonitor)(nil) -) +var _ pkg.SystemMonitor = (*systemMonitor)(nil) // systemMonitor stream type systemMonitor struct { @@ -199,3 +198,21 @@ func (m *systemMonitor) Nics(ctx context.Context) <-chan pkg.NicsIOCounterStat { return ch } + +// Get the types of workloads can be deployed depending on the network manager running on the node +func (n *systemMonitor) GetNodeFeatures() []pkg.NodeFeature { + feat := []pkg.NodeFeature{ + pkg.NodeFeature(zos.ZMountType), + pkg.NodeFeature(zos.NetworkType), + pkg.NodeFeature(zos.ZDBType), + pkg.NodeFeature(zos.ZMachineType), + pkg.NodeFeature(zos.VolumeType), + pkg.NodeFeature(zos.PublicIPv4Type), + pkg.NodeFeature(zos.PublicIPType), + pkg.NodeFeature(zos.GatewayNameProxyType), + pkg.NodeFeature(zos.GatewayFQDNProxyType), + pkg.NodeFeature(zos.QuantumSafeFSType), + pkg.NodeFeature(zos.ZLogsType), + } + return feat +} diff --git a/pkg/network/mycelium/mycelium.go b/pkg/network/mycelium/mycelium.go index 482922029..4caff618a 100644 --- a/pkg/network/mycelium/mycelium.go +++ b/pkg/network/mycelium/mycelium.go @@ -183,7 +183,7 @@ func (s *MyceliumServer) InspectMycelium() (inspection MyceliumInspection, err e // IP return the address in the 400::/7 subnet allocated by mycelium func (m *MyceliumInspection) IP() net.IP { - return net.IP(m.Address) + return m.Address } // Subnet return the 400::/64 subnet allocated by mycelium diff --git a/pkg/network/network_test.go b/pkg/network/network_test.go index 1d6ac47fd..6c08d15e8 100644 --- a/pkg/network/network_test.go +++ b/pkg/network/network_test.go @@ -30,7 +30,8 @@ func TestKeys(t *testing.T) { strEncrypted := fmt.Sprintf("%x", encrypted) strDecrypted := "" - fmt.Sscanf(strEncrypted, "%x", &strDecrypted) + _, err = fmt.Sscanf(strEncrypted, "%x", &strDecrypted) + require.NoError(t, err) decrypted, err := crypto.Decrypt([]byte(strDecrypted), sk) require.NoError(t, err) diff --git a/pkg/network/networker.go b/pkg/network/networker.go index ccef4a6d0..34ba6bc20 100644 --- a/pkg/network/networker.go +++ b/pkg/network/networker.go @@ -360,7 +360,7 @@ func (n *networker) SetupPrivTap(networkID pkg.NetID, name string) (ifc string, } func (n *networker) TapExists(name string) (bool, error) { - log.Info().Str("tap-name", string(name)).Msg("Checking if tap interface exists") + log.Info().Str("tap-name", name).Msg("Checking if tap interface exists") tapIface, err := tapName(name) if err != nil { @@ -372,7 +372,7 @@ func (n *networker) TapExists(name string) (bool, error) { // RemoveTap in the network resource. func (n *networker) RemoveTap(name string) error { - log.Info().Str("tap-name", string(name)).Msg("Removing tap interface") + log.Info().Str("tap-name", name).Msg("Removing tap interface") tapIface, err := tapName(name) if err != nil { @@ -390,7 +390,7 @@ func (n *networker) PublicIPv4Support() bool { // reservation id. It is hooked to the public bridge. The name of the tap // interface is returned func (n *networker) SetupPubTap(name string) (string, error) { - log.Info().Str("pubtap-name", string(name)).Msg("Setting up public tap interface") + log.Info().Str("pubtap-name", name).Msg("Setting up public tap interface") if !n.ndmz.SupportsPubIPv4() { return "", errors.New("can't create public tap on this node") @@ -408,7 +408,7 @@ func (n *networker) SetupPubTap(name string) (string, error) { // SetupMyceliumTap creates a new mycelium tap device attached to this network resource with deterministic IP address func (n *networker) SetupMyceliumTap(name string, netID zos.NetID, config zos.MyceliumIP) (tap pkg.PlanetaryTap, err error) { - log.Info().Str("tap-name", string(name)).Msg("Setting up mycelium tap interface") + log.Info().Str("tap-name", name).Msg("Setting up mycelium tap interface") network, err := n.networkOf(netID) if err != nil { @@ -452,7 +452,7 @@ func (n *networker) SetupMyceliumTap(name string, netID zos.NetID, config zos.My // SetupYggTap sets up a tap device in the host namespace for the yggdrasil ip func (n *networker) SetupYggTap(name string) (tap pkg.PlanetaryTap, err error) { - log.Info().Str("tap-name", string(name)).Msg("Setting up yggdrasil tap interface") + log.Info().Str("tap-name", name).Msg("Setting up yggdrasil tap interface") tapIface, err := tapName(name) if err != nil { @@ -486,7 +486,7 @@ func (n *networker) SetupYggTap(name string) (tap pkg.PlanetaryTap, err error) { // PubTapExists checks if the tap device for the public network exists already func (n *networker) PubTapExists(name string) (bool, error) { - log.Info().Str("pubtap-name", string(name)).Msg("Checking if public tap interface exists") + log.Info().Str("pubtap-name", name).Msg("Checking if public tap interface exists") tapIface, err := pubTapName(name) if err != nil { @@ -499,7 +499,7 @@ func (n *networker) PubTapExists(name string) (bool, error) { // RemovePubTap removes the public tap device from the host namespace // of the networkID func (n *networker) RemovePubTap(name string) error { - log.Info().Str("pubtap-name", string(name)).Msg("Removing public tap interface") + log.Info().Str("pubtap-name", name).Msg("Removing public tap interface") tapIface, err := pubTapName(name) if err != nil { @@ -660,7 +660,7 @@ func (n *networker) RemovePubIPFilter(filterName string) error { // DisconnectPubTap disconnects the public tap from the network. The interface // itself is not removed and will need to be cleaned up later func (n *networker) DisconnectPubTap(name string) error { - log.Info().Str("pubtap-name", string(name)).Msg("Disconnecting public tap interface") + log.Info().Str("pubtap-name", name).Msg("Disconnecting public tap interface") tapIfaceName, err := pubTapName(name) if err != nil { return errors.Wrap(err, "could not get network namespace tap device name") diff --git a/pkg/network/nr/net_resource.go b/pkg/network/nr/net_resource.go index ac5088c84..43b126fa6 100644 --- a/pkg/network/nr/net_resource.go +++ b/pkg/network/nr/net_resource.go @@ -644,7 +644,7 @@ func (nr *NetResource) wgPeers() ([]*wireguard.Peer, error) { } wgPeer := &wireguard.Peer{ - PublicKey: string(peer.WGPublicKey), + PublicKey: peer.WGPublicKey, AllowedIPs: allowedIPs, Endpoint: peer.Endpoint, } diff --git a/pkg/perf/graphql/graphql_nodes.go b/pkg/perf/graphql/graphql_nodes.go index b687f4f21..75752a340 100644 --- a/pkg/perf/graphql/graphql_nodes.go +++ b/pkg/perf/graphql/graphql_nodes.go @@ -2,6 +2,7 @@ package graphql import ( "context" + "errors" "fmt" "math/rand" "time" @@ -16,13 +17,15 @@ const ( // GraphQl for tf graphql client type GraphQl struct { - client *graphql.Client + urls []string } // NewGraphQl creates a new tf graphql client -func NewGraphQl(url string) GraphQl { - client := graphql.NewClient(url, nil) - return GraphQl{client: client} +func NewGraphQl(urls ...string) (GraphQl, error) { + if len(urls) == 0 { + return GraphQl{}, errors.New("urls can't be empty") + } + return GraphQl{urls: urls}, nil } // Node from graphql @@ -85,7 +88,7 @@ func (g *GraphQl) GetUpNodes(ctx context.Context, nodesNum int, farmID, excludeF Nodes []Node }{} - if err := g.client.Exec(ctx, query, &res, nil); err != nil { + if err := g.exec(ctx, query, &res, nil); err != nil { return []Node{}, err } @@ -101,9 +104,20 @@ func (g *GraphQl) getItemTotalCount(ctx context.Context, itemName string, option } }{} - if err := g.client.Exec(ctx, query, &res, nil); err != nil { + if err := g.exec(ctx, query, &res, nil); err != nil { return 0, err } return res.Items.Count, nil } + +// exec is a wrapper around graphql.Client.Exec to retry another endpoints in case some are down. +func (g *GraphQl) exec(ctx context.Context, query string, result interface{}, variables map[string]interface{}, options ...graphql.Option) (err error) { + for _, url := range g.urls { + client := graphql.NewClient(url, nil) + if err = client.Exec(ctx, query, result, variables, options...); err == nil { + return + } + } + return +} diff --git a/pkg/perf/healthcheck/network.go b/pkg/perf/healthcheck/network.go index 1ab10c89c..9a99eb49a 100644 --- a/pkg/perf/healthcheck/network.go +++ b/pkg/perf/healthcheck/network.go @@ -17,10 +17,10 @@ const defaultRequestTimeout = 5 * time.Second func networkCheck(ctx context.Context) []error { env := environment.MustGet() - servicesUrl := []string{ - env.ActivationURL, env.GraphQL, env.FlistURL, - } + servicesUrl := []string{env.FlistURL} + servicesUrl = append(append(servicesUrl, env.SubstrateURL...), env.RelayURL...) + servicesUrl = append(append(servicesUrl, env.ActivationURL...), env.GraphQL...) var errors []error diff --git a/pkg/perf/healthcheck/ntp.go b/pkg/perf/healthcheck/ntp.go index 0effa086a..594524587 100644 --- a/pkg/perf/healthcheck/ntp.go +++ b/pkg/perf/healthcheck/ntp.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + "github.com/cenkalti/backoff/v3" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg/zinit" @@ -17,8 +18,13 @@ const acceptableSkew = 10 * time.Minute func RunNTPCheck(ctx context.Context) { go func() { for { - if err := ntpCheck(); err != nil { + exp := backoff.NewExponentialBackOff() + retryNotify := func(err error, d time.Duration) { log.Error().Err(err).Msg("failed to run ntp check") + } + + if err := backoff.RetryNotify(ntpCheck, backoff.WithContext(exp, ctx), retryNotify); err != nil { + log.Error().Err(err).Send() continue } diff --git a/pkg/perf/iperf/iperf_task.go b/pkg/perf/iperf/iperf_task.go index 002b525d6..bc12ad1b2 100644 --- a/pkg/perf/iperf/iperf_task.go +++ b/pkg/perf/iperf/iperf_task.go @@ -76,7 +76,10 @@ func (t *IperfTest) Jitter() uint32 { // Run runs the tcp test and returns the result func (t *IperfTest) Run(ctx context.Context) (interface{}, error) { env := environment.MustGet() - g := graphql.NewGraphQl(env.GraphQL) + g, err := graphql.NewGraphQl(env.GraphQL...) + if err != nil { + return nil, err + } // get public up nodes freeFarmNodes, err := g.GetUpNodes(ctx, 0, 1, 0, true, true) @@ -151,7 +154,7 @@ func (t *IperfTest) runIperfTest(ctx context.Context, clientIP string, tcp bool) operation := func() error { res := runIperfCommand(ctx, opts) if res.Error == errServerBusy { - return fmt.Errorf(errServerBusy) + return errors.New(errServerBusy) } report = res diff --git a/pkg/perf/publicip/publicip_task.go b/pkg/perf/publicip/publicip_task.go index 5b68c1740..51595cecc 100644 --- a/pkg/perf/publicip/publicip_task.go +++ b/pkg/perf/publicip/publicip_task.go @@ -34,11 +34,15 @@ const ( FetchRealIPFailed = "failed to get real public IP to the node" ) -var errPublicIPLookup = errors.New("failed to reach public ip service") -var errSkippedValidating = errors.New("skipped, there is a node with less ID available") +var ( + errPublicIPLookup = errors.New("failed to reach public ip service") + errSkippedValidating = errors.New("skipped, there is a node with less ID available") +) -const testMacvlan = "pub" -const testNamespace = "pubtestns" +const ( + testMacvlan = "pub" + testNamespace = "pubtestns" +) type publicIPValidationTask struct{} @@ -96,7 +100,6 @@ func (p *publicIPValidationTask) Run(ctx context.Context) (interface{}, error) { report, err = p.validateIPs(farm.PublicIPs) return err }) - if err != nil { return nil, fmt.Errorf("failed to run public IP validation: %w", err) } @@ -179,7 +182,10 @@ func (p *publicIPValidationTask) validateIPs(publicIPs []substrate.PublicIP) (ma func isLeastValidNode(ctx context.Context, farmID uint32, substrateGateway *stubs.SubstrateGatewayStub) (bool, error) { env := environment.MustGet() - gql := graphql.NewGraphQl(env.GraphQL) + gql, err := graphql.NewGraphQl(env.GraphQL...) + if err != nil { + return false, err + } nodes, err := gql.GetUpNodes(ctx, 0, farmID, 0, false, false) if err != nil { @@ -196,13 +202,12 @@ func isLeastValidNode(ctx context.Context, farmID uint32, substrateGateway *stub } return nil }, backoff.NewConstantBackOff(10*time.Second)) - if err != nil { return false, fmt.Errorf("failed to get node id: %w", err) } for _, node := range nodes { - if node.NodeID >= uint32(nodeID) { + if node.NodeID >= nodeID { continue } n, err := substrateGateway.GetNode(ctx, node.NodeID) diff --git a/pkg/power/ethtool.go b/pkg/power/ethtool.go index 85a28a51f..6480ed8c6 100644 --- a/pkg/power/ethtool.go +++ b/pkg/power/ethtool.go @@ -57,7 +57,7 @@ func valueOfFlag(output []byte, flag Flag) (string, error) { if len(parts) != 2 { return "", fmt.Errorf("invalid ethtool output format (%s)", line) } - return strings.TrimSpace(string(parts[1])), nil + return strings.TrimSpace(parts[1]), nil } return "", ErrFlagNotFound diff --git a/pkg/provision/auth.go b/pkg/provision/auth.go index 6a6ae0ffd..80eca8669 100644 --- a/pkg/provision/auth.go +++ b/pkg/provision/auth.go @@ -40,7 +40,7 @@ func (s *substrateTwins) GetKey(id uint32) ([]byte, error) { key := user.Account.PublicKey() s.mem.Add(id, key) - return []byte(key), nil + return key, nil } type substrateAdmins struct { diff --git a/pkg/provision/engine.go b/pkg/provision/engine.go index 5880fa27e..42a4a2d36 100644 --- a/pkg/provision/engine.go +++ b/pkg/provision/engine.go @@ -572,7 +572,7 @@ func (e *NativeEngine) validate(ctx context.Context, dl *gridtypes.Deployment, n return ctx, fmt.Errorf("substrate is not configured in engine") } - contract, subErr := e.substrateGateway.GetContract(ctx, uint64(dl.ContractID)) + contract, subErr := e.substrateGateway.GetContract(ctx, dl.ContractID) if subErr.IsError() { return nil, errors.Wrap(subErr.Err, "failed to get deployment contract") } diff --git a/pkg/provision/storage.fs/storage.go b/pkg/provision/storage.fs/storage.go index ffb630fad..ee32085ac 100644 --- a/pkg/provision/storage.fs/storage.go +++ b/pkg/provision/storage.fs/storage.go @@ -283,7 +283,7 @@ func (s *Fs) byTwin(twin uint32) ([]uint64, error) { continue } - ids = append(ids, uint64(id)) + ids = append(ids, id) } return ids, nil diff --git a/pkg/qsfsd/cleanup.go b/pkg/qsfsd/cleanup.go index 2a8a7a140..4c7f16301 100644 --- a/pkg/qsfsd/cleanup.go +++ b/pkg/qsfsd/cleanup.go @@ -59,11 +59,11 @@ func (q *QSFS) checkDeadQSFSs(ctx context.Context, state *failedQSFSState) error } for _, path := range paths { wlID := filepath.Base(path) - metrics, err := q.qsfsMetrics(ctx, string(wlID)) + metrics, err := q.qsfsMetrics(ctx, wlID) if err != nil { - log.Err(err).Str("id", string(wlID)).Msg("couldn't get qsfs metrics") - state.metricsFailureCount[string(wlID)] += 1 - if state.metricsFailureCount[string(wlID)] >= 10 { + log.Err(err).Str("id", wlID).Msg("couldn't get qsfs metrics") + state.metricsFailureCount[wlID] += 1 + if state.metricsFailureCount[wlID] >= 10 { q.Unmount(wlID) state.delete(wlID) } diff --git a/pkg/qsfsd/qsfs.go b/pkg/qsfsd/qsfs.go index a77db2bc6..387fc1d49 100644 --- a/pkg/qsfsd/qsfs.go +++ b/pkg/qsfsd/qsfs.go @@ -169,7 +169,7 @@ func (q *QSFS) Mount(wlID string, cfg zos.QuantumSafeFS) (info pkg.QSFSInfo, err if containerErr != nil { log.Error().Err(containerErr).Msg("Failed to read container logs") } - err = errors.Wrapf(lerr, fmt.Sprintf("Container Logs:\n%s", logs)) + err = errors.Wrapf(lerr, "Container Logs:\n%s", logs) return } log.Debug().Str("duration", time.Since(t).String()).Msg("waiting for qsfs deployment took") diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 91b3ba66d..30b6ddd31 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -416,7 +416,7 @@ func TestCacheResize(t *testing.T) { Excl: 1, }, } - vol.On("Limit", uint64(cacheSize)).Return(nil) + vol.On("Limit", cacheSize).Return(nil) err := m.checkAndResizeCache(&vol, cacheSize) require.NoError(t, err) @@ -444,7 +444,7 @@ func TestCacheResize(t *testing.T) { Excl: 0, // no files }, } - vol.On("Limit", uint64(cacheSize)).Return(nil) + vol.On("Limit", cacheSize).Return(nil) err = m.checkAndResizeCache(&vol, cacheSize) require.NoError(t, err) @@ -468,7 +468,7 @@ func TestCacheResize(t *testing.T) { Excl: 91, }, } - vol.On("Limit", uint64(100+cacheSize)).Return(nil) + vol.On("Limit", 100+cacheSize).Return(nil) err = m.checkAndResizeCache(&vol, cacheSize) require.NoError(t, err) diff --git a/pkg/stubs/api_gateway_stub.go b/pkg/stubs/api_gateway_stub.go index 1fbdf0984..dbd340cfa 100644 --- a/pkg/stubs/api_gateway_stub.go +++ b/pkg/stubs/api_gateway_stub.go @@ -63,7 +63,7 @@ func (s *SubstrateGatewayStub) CreateTwin(ctx context.Context, arg0 string, arg1 return } -func (s *SubstrateGatewayStub) EnsureAccount(ctx context.Context, arg0 string, arg1 string, arg2 string) (ret0 tfchainclientgo.AccountInfo, ret1 error) { +func (s *SubstrateGatewayStub) EnsureAccount(ctx context.Context, arg0 []string, arg1 string, arg2 string) (ret0 tfchainclientgo.AccountInfo, ret1 error) { args := []interface{}{arg0, arg1, arg2} result, err := s.client.RequestContext(ctx, s.module, s.object, "EnsureAccount", args...) if err != nil { diff --git a/pkg/stubs/system_monitor_stub.go b/pkg/stubs/system_monitor_stub.go index b98bd9ae3..2ffc04907 100644 --- a/pkg/stubs/system_monitor_stub.go +++ b/pkg/stubs/system_monitor_stub.go @@ -75,6 +75,22 @@ func (s *SystemMonitorStub) Disks(ctx context.Context) (<-chan pkg.DisksIOCounte return ch, nil } +func (s *SystemMonitorStub) GetNodeFeatures(ctx context.Context) (ret0 []pkg.NodeFeature) { + args := []interface{}{} + result, err := s.client.RequestContext(ctx, s.module, s.object, "GetNodeFeatures", args...) + if err != nil { + panic(err) + } + result.PanicOnError() + loader := zbus.Loader{ + &ret0, + } + if err := result.Unmarshal(&loader); err != nil { + panic(err) + } + return +} + func (s *SystemMonitorStub) Memory(ctx context.Context) (<-chan pkg.VirtualMemoryStat, error) { ch := make(chan pkg.VirtualMemoryStat, 1) recv, err := s.client.Stream(ctx, s.module, s.object, "Memory") diff --git a/pkg/substrate_gateway/substrate_gateway.go b/pkg/substrate_gateway/substrate_gateway.go index 7bd936970..8a7d7fe7e 100644 --- a/pkg/substrate_gateway/substrate_gateway.go +++ b/pkg/substrate_gateway/substrate_gateway.go @@ -48,16 +48,25 @@ func (g *substrateGateway) CreateTwin(relay string, pk []byte) (uint32, error) { return g.sub.CreateTwin(g.identity, relay, pk) } -func (g *substrateGateway) EnsureAccount(activationURL string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) { +func (g *substrateGateway) EnsureAccount(activationURL []string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) { log.Debug(). Str("method", "EnsureAccount"). - Str("activation url", activationURL). + Strs("activation url", activationURL). Str("terms and conditions link", termsAndConditionsLink). Str("terms and conditions hash", termsAndConditionsHash). Msg("method called") g.mu.Lock() defer g.mu.Unlock() - return g.sub.EnsureAccount(g.identity, activationURL, termsAndConditionsLink, termsAndConditionsHash) + + for _, url := range activationURL { + info, err = g.sub.EnsureAccount(g.identity, url, termsAndConditionsLink, termsAndConditionsHash) + // check other activationURL only if EnsureAccount failed with ActivationServiceError + if err == nil || !errors.As(err, &substrate.ActivationServiceError{}) { + return + } + log.Debug().Str("activation url", url).Err(err).Msg("failed to EnsureAccount with ActivationServiceError") + } + return } func (g *substrateGateway) GetContract(id uint64) (result substrate.Contract, serr pkg.SubstrateError) { diff --git a/pkg/vm/ch.go b/pkg/vm/ch.go index 30ed5ac8b..22e068538 100644 --- a/pkg/vm/ch.go +++ b/pkg/vm/ch.go @@ -286,6 +286,7 @@ func (m *Machine) waitAndAdjOom(ctx context.Context, name string, socket string) func (m *Machine) startFs(socket, path string) (int, error) { cmd := exec.Command("busybox", "setsid", "virtiofsd-rs", + "--xattr", "--socket-path", socket, "--shared-dir", path, "--shared-dir-stats", fmt.Sprintf("/usr/share/btrfs/volstat.sh %s", path), diff --git a/pkg/vm/manager.go b/pkg/vm/manager.go index 42524dac2..8a9aa51f9 100644 --- a/pkg/vm/manager.go +++ b/pkg/vm/manager.go @@ -322,7 +322,7 @@ func (m *Module) withLogs(path string, err error) error { return errors.Wrapf(err, "failed to tail machine logs: %s", tailErr) } - return errors.Wrapf(err, string(logs)) + return errors.Wrap(err, logs) } func (m *Module) checkDevicesUsed(devices []string) error { diff --git a/pkg/zdb/admin.go b/pkg/zdb/admin.go index 3832bf274..84f3a4482 100644 --- a/pkg/zdb/admin.go +++ b/pkg/zdb/admin.go @@ -1,7 +1,7 @@ package zdb import ( - "fmt" + "errors" "strings" "github.com/gomodule/redigo/redis" @@ -18,7 +18,7 @@ func (c *clientImpl) CreateNamespace(name string) error { return err } if ok != "OK" { - return fmt.Errorf(ok) + return errors.New(ok) } return nil } @@ -49,7 +49,7 @@ func (c *clientImpl) DeleteNamespace(name string) error { return err } if ok != "OK" { - return fmt.Errorf(ok) + return errors.New(ok) } return nil } @@ -70,7 +70,7 @@ func (c *clientImpl) NamespaceSetSize(name string, size uint64) error { return err } if ok != "OK" { - return fmt.Errorf(ok) + return errors.New(ok) } return nil } @@ -84,7 +84,7 @@ func (c *clientImpl) NamespaceSetPassword(name, password string) error { return err } if ok != "OK" { - return fmt.Errorf(ok) + return errors.New(ok) } return nil } @@ -98,7 +98,7 @@ func (c *clientImpl) NamespaceSetMode(name, mode string) error { return err } if ok != "OK" { - return fmt.Errorf(ok) + return errors.New(ok) } return nil } @@ -116,7 +116,7 @@ func (c *clientImpl) NamespaceSetLock(name string, lock bool) error { return err } if ok != "OK" { - return fmt.Errorf(ok) + return errors.New(ok) } return nil } @@ -136,7 +136,7 @@ func (c *clientImpl) NamespaceSetPublic(name string, public bool) error { return err } if ok != "OK" { - return fmt.Errorf(ok) + return errors.New(ok) } return nil } diff --git a/pkg/zos_api/routes.go b/pkg/zos_api/routes.go index 7fa70493a..8006f23b7 100644 --- a/pkg/zos_api/routes.go +++ b/pkg/zos_api/routes.go @@ -5,7 +5,6 @@ import ( ) func (g *ZosAPI) SetupRoutes(router *peer.Router) { - root := router.SubRoute("zos") root.Use(g.log) system := root.SubRoute("system") @@ -13,6 +12,7 @@ func (g *ZosAPI) SetupRoutes(router *peer.Router) { system.WithHandler("dmi", g.systemDMIHandler) system.WithHandler("hypervisor", g.systemHypervisorHandler) system.WithHandler("diagnostics", g.systemDiagnosticsHandler) + system.WithHandler("node_features_get", g.systemNodeFeaturesHandler) perf := root.SubRoute("perf") perf.WithHandler("get", g.perfGetHandler) diff --git a/pkg/zos_api/system.go b/pkg/zos_api/system.go index 7bae7beb6..88b775177 100644 --- a/pkg/zos_api/system.go +++ b/pkg/zos_api/system.go @@ -37,3 +37,7 @@ func (g *ZosAPI) systemHypervisorHandler(ctx context.Context, payload []byte) (i func (g *ZosAPI) systemDiagnosticsHandler(ctx context.Context, payload []byte) (interface{}, error) { return g.diagnosticsManager.GetSystemDiagnostics(ctx) } + +func (g *ZosAPI) systemNodeFeaturesHandler(ctx context.Context, payload []byte) (interface{}, error) { + return g.systemMonitorStub.GetNodeFeatures(ctx), nil +} diff --git a/pkg/zos_api/zos_api.go b/pkg/zos_api/zos_api.go index 3e21ef61b..e3661f706 100644 --- a/pkg/zos_api/zos_api.go +++ b/pkg/zos_api/zos_api.go @@ -14,6 +14,7 @@ import ( type ZosAPI struct { oracle *capacity.ResourceOracle versionMonitorStub *stubs.VersionMonitorStub + systemMonitorStub *stubs.SystemMonitorStub provisionStub *stubs.ProvisionStub networkerStub *stubs.NetworkerStub statisticsStub *stubs.StatisticsStub @@ -37,6 +38,7 @@ func NewZosAPI(manager substrate.Manager, client zbus.Client, msgBrokerCon strin api := ZosAPI{ oracle: capacity.NewResourceOracle(storageModuleStub), versionMonitorStub: stubs.NewVersionMonitorStub(client), + systemMonitorStub: stubs.NewSystemMonitorStub(client), provisionStub: stubs.NewProvisionStub(client), networkerStub: stubs.NewNetworkerStub(client), statisticsStub: stubs.NewStatisticsStub(client), diff --git a/scripts/debug_image.md b/scripts/debug_image.md index 31d3002ec..19f070b62 100644 --- a/scripts/debug_image.md +++ b/scripts/debug_image.md @@ -8,15 +8,18 @@ The debug image script facilitates development and debugging of flists runs on Z Either during development, specify a directory containing the rootfs. Or to debug existing flist, pass an flist url. +> NOTE: the script can't be run on a vm on the grid due to nested virtualization isn't supported +> NOTE: script must run as a superuser + ```bash -# run image from a directory -./debug_image.sh --image /tmp/rootfs +# enter a sudo shell +sudo -i -# run image from a directory with a create cidata image -./debug_image.sh --image /tmp/rootfs --debug true --cidata /tmp/cloud-init.img +# install deps +./install_deps -# run image from an flist with login cred. `foo:bar` -./debug_image.sh --image https://hub.grid.tf/tf-official-apps/discourse-v4.0.flist --init /start.sh --user foo --pass bar +# run image from a directory +./debug_image.sh -d --image https://hub.grid.tf/omarabdulaziz.3bot/ubuntu-jammy.flist ``` ## Image Types @@ -30,9 +33,6 @@ Either during development, specify a directory containing the rootfs. Or to debu ## Flags - `--image`: [REQUIRED] directory or flist url -- `--d`: enables `set -x` in the bash script -- `--h`: show the help message -- `--c`: run the image in container mode, will provide kernel/initrd from cloud-container - `--kernel`: kernel file path (compressed or uncompressed). default: `/boot/vmlinuz` - `--initramfs`: Initrd image path. default: `/boot/initrd.img` - `--init`: entrypoint for the machine. @@ -40,6 +40,10 @@ Either during development, specify a directory containing the rootfs. Or to debu - `--user`: cloud-init username. default is user - `--pass`: cloud-init password. default is pass - `--name`: cloud-init machine name. default is cloud +- `-d`: enables `set -x` in the bash script +- `-h`: show the help message +- `-c`: run the image in container mode, will provide kernel/initrd from cloud-container +- `-i`: install any missing deps NOTE: @@ -52,6 +56,18 @@ NOTE: - [rfs v1](https://github.com/threefoldtech/rfs/tree/v1): mounts the flist file into a directory serving as the lower layer of the overlay file system. - `overlayfs`: mounts a read-write layer on the rootfs +## Install dependencies + +Use the `install_deps.sh` script to install the needed binaries. + +- cloud-hypervisor: required +- virtiofsd: required +- rfs1: required +- mkdosfs, mcopy: needed to create cidata image +- rust compiler and cargo: needed to build virtiofsd + +> NOTE: script must run as a superuser + ## Script Walkthrough - **Validation:** @@ -74,48 +90,20 @@ NOTE: - Kills all attached processes. - Unmounts and clears directories. -## Install dependencies - -- **cloud-hypervisor** - - ```bash - git clone https://github.com/cloud-hypervisor/cloud-hypervisor.git - cd cloud-hypervisor - cargo build --release - sudo setcap cap_net_admin+ep ./target/release/cloud-hypervisor +## Control vm - sudo ln -s $(realpath ./target/release/cloud-hypervisor) /usr/local/bin/cloud-hypervisor - ``` +a `ch-remote` is a very useful tool that can control cloud-hypervisor on the runtime. check the [docs](https://www.cloudhypervisor.org/docs/prologue/commands/#ch-remote-binary) -- **virtiofsd** +Install - ```bash - git clone https://gitlab.com/muhamad.azmy/virtiofsd.git - cd virtiofsd - cargo build --release - sudo setcap cap_net_admin+ep ./target/release/virtiofsd - - sudo ln -s $(realpath ./target/release/virtiofsd) /usr/local/bin/virtiofsd - ``` - -- **rfs** - - ```bash - wget https://github.com/threefoldtech/rfs/releases/download/v1.1.1/rfs - chmod +x ./rfs - - sudo ln -s $(realpath ./rfs) /usr/local/bin/rfs1 - ``` - -- **mkdosfs** - only needed if the script gonna create the cidata image. - - ```bash - apt-get install dosfstools - ``` +```bash +wget https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v36.0/ch-remote +chmod +x ch-remote +``` -- **screen** +Usage - ```bash - apt-get install screen - ``` +```bash +# connect with the api socket connected plugged in cloud-hypervisor. found it on the script +sudo ./ch-remote --api-socket=/tmp/ch-sock shutdown-vmm +``` diff --git a/scripts/debug_image.sh b/scripts/debug_image.sh index af1666adf..fbe9b01fb 100755 --- a/scripts/debug_image.sh +++ b/scripts/debug_image.sh @@ -2,6 +2,7 @@ # Constants readonly SOCKET="/tmp/virtiofs.sock" +readonly CH_API_SOCKET="/tmp/ch-api.sock" readonly OVERLAYFS="/tmp/overlay" readonly CCFLIST="https://hub.grid.tf/tf-autobuilder/cloud-container-9dba60e.flist" readonly MACHINE_TYPE="machine" @@ -24,6 +25,7 @@ declare -A options=( ) image_type=$MACHINE_TYPE +install_deps=false init="" kernel="" initramfs="" @@ -51,6 +53,7 @@ usage() { echo " -d Enable debugging mode with 'set -x'." echo " -h Show this help message." echo " -c Run on container mode (will provide a kernel and initrd)." + echo " -i Don't fail on missing deps and install them." echo "" echo "Example:" echo " debug_image.sh --image /path/to/rootfs --debug true" @@ -75,6 +78,10 @@ handle_options() { image_type=$CONTAINER_TYPE shift ;; + -i) + install_deps=true + shift + ;; *) usage exit 1 @@ -88,37 +95,23 @@ handle_options() { } check_or_install_deps() { - mkdir -p ~/chv - cd ~/chv - - if ! command -v cloud-hypervisor &>/dev/null; then - wget https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v39.0/cloud-hypervisor - chmod +x cloud-hypervisor - sudo ln -s $(realpath ./cloud-hypervisor) /usr/local/bin/cloud-hypervisor - fi - - if ! command -v virtiofsd &>/dev/null; then - git clone https://gitlab.com/muhamad.azmy/virtiofsd.git - pushd virtiofsd - cargo build --release - sudo ln -s $(realpath ./target/release/virtiofsd) /usr/local/bin/virtiofsd - popd - fi - - if ! command -v rfs1 &>/dev/null; then - wget https://github.com/threefoldtech/rfs/releases/download/v1.1.1/rfs - chmod +x rfs - sudo ln -s $(realpath ./rfs) /usr/local/bin/rfs1 + if [ "$install_deps" = true ]; then + bash ./install_deps.sh + return fi - if [[ -n "${options[--cidata]}" && ! $(command -v mkdosfs &>/dev/null) ]]; then - sudo apt update - sudo apt -y install dosfstools - fi + for cmd in cloud-hypervisor virtiofsd rfs1 screen; do + if ! command -v $cmd &>/dev/null; then + fail "$cmd command not found" + fi + done - if ! command -v screen &>/dev/null; then - sudo apt update - sudo apt -y install screen + if [[ -n "${options[--cidata]}" ]]; then + for cmd in mcopy mkdosfs; do + if ! command -v $cmd &>/dev/null; then + fail "$cmd command not found" + fi + done fi } @@ -195,7 +188,7 @@ mount_flist() { wget $flist_url -O $flist_path fi - sudo mkdir -p "$mountpoint" + mkdir -p "$mountpoint" rfs1 --meta "$flist_path" "$mountpoint" & pids+=($!) @@ -225,8 +218,8 @@ prepare_rootfs() { fi echo "Mounting overlay" - sudo mkdir -p /tmp/upper /tmp/workdir "$OVERLAYFS" - sudo mount \ + mkdir -p /tmp/upper /tmp/workdir "$OVERLAYFS" + mount \ -t overlay \ -o lowerdir="$lowerdir",upperdir=/tmp/upper,workdir=/tmp/workdir \ none \ @@ -234,7 +227,7 @@ prepare_rootfs() { echo "Starting virtiofs" # a trick to not mess the logs, it asks for sudo - sudo screen -dmS virtiofsd_session sudo virtiofsd --socket-path="$SOCKET" --shared-dir="$OVERLAYFS" --cache=never + screen -dmS virtiofsd_session sudo virtiofsd --socket-path="$SOCKET" --shared-dir="$OVERLAYFS" --cache=never pids+=($!) } @@ -243,8 +236,8 @@ cleanup() { dirs=("$OVERLAYFS" /tmp/flist /tmp/cloud-container /tmp/upper /tmp/workdir) for dir in "${dirs_to_umount[@]}"; do - sudo umount "$dir" &>/dev/null || true - sudo rm -rf "$dir" &>/dev/null || true + umount "$dir" &>/dev/null || true + rm -rf "$dir" &>/dev/null || true done for pid in "${pids[@]}"; do @@ -255,7 +248,7 @@ cleanup() { } boot() { - sudo cloud-hypervisor \ + cloud-hypervisor \ --cpus boot=1,max=1 \ --memory size=1024M,shared=on \ --kernel "${kernel}" \ @@ -263,13 +256,15 @@ boot() { --fs tag=vroot,socket="$SOCKET" \ --disk path="$cidata" \ --cmdline "$cmdline" \ + --api-socket="$CH_API_SOCKET" \ --serial tty \ --console off } -if [ "$EUID" -ne 0 ] - then echo "Please run as root" - exit +# must run as superuser +if [ $(id -u) != "0" ]; then +echo "You must be the superuser to run this script" >&2 +exit 1 fi cleanup diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh new file mode 100755 index 000000000..7c0bedda2 --- /dev/null +++ b/scripts/install_deps.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +CHV_VERSION="v39.0" +CHV_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${CHV_VERSION}/cloud-hypervisor" +RUSTUP_URL="https://sh.rustup.rs" +VIRTIOFSD_REPO="https://gitlab.com/muhamad.azmy/virtiofsd/-/jobs/6547244336/artifacts/download?file_type=archive" +RFS_VERSION="v1.1.1" +RFS_URL="https://github.com/threefoldtech/rfs/releases/download/${RFS_VERSION}/rfs" + +install_chv() { + echo "Installing cloud-hypervisor ${CHV_VERSION} ..." + wget -q ${CHV_URL} -O /usr/local/bin/cloud-hypervisor + chmod +x /usr/local/bin/cloud-hypervisor + setcap cap_sys_admin,cap_dac_override+eip /usr/local/bin/cloud-hypervisor +} + + +install_virtiofsd() { + echo "Installing virtiofsd ..." + + # specially needed for virtiofsd bin + apt -y update + apt -y install libseccomp-dev libcap-ng-dev + + curl -L -k -o "/tmp/virtiofsd-rs.zip" ${VIRTIOFSD_REPO} + unzip -p /tmp/virtiofsd-rs.zip > /usr/local/bin/virtiofsd + chmod +x /usr/local/bin/virtiofsd + setcap cap_sys_admin,cap_dac_override+eip /usr/local/bin/virtiofsd +} + +install_rfs() { + echo "Installing rfs ${RFS_VERSION} ..." + wget -q ${RFS_URL} + chmod +x rfs + mv ./rfs /usr/local/bin/rfs1 +} + +main() { + # must run as superuser + if [ $(id -u) != "0" ]; then + echo "You must be the superuser to run this script" >&2 + exit 1 + fi + + TEMP_DIR=$(mktemp -d) + pushd $TEMP_DIR + + # install cloud-hypervisor + if ! command -v cloud-hypervisor &>/dev/null; then + install_chv + fi + + # install virtiofsd + if ! command -v virtiofsd &>/dev/null; then + + install_virtiofsd + fi + + # install rfs + if ! command -v rfs1 &>/dev/null; then + install_rfs + fi + + # install mcopy/mkdosfs needed to create cidata image. comment if you a have the image + apt -y install dosfstools mtools fuse + + # install screen for managing multiple servers + # NOTE: rust bins like virtiofsd miss the logs, runs on a screen session to workaround that + apt -y install screen + + popd + rm -rf $TEMP_DIR +} + +main \ No newline at end of file diff --git a/tools/zos-update-worker/Makefile b/tools/zos-update-worker/Makefile index 70d5e2dbd..9714904a3 100644 --- a/tools/zos-update-worker/Makefile +++ b/tools/zos-update-worker/Makefile @@ -8,41 +8,14 @@ ldflags='-w -s -X $(version).Branch=$(branch) -X $(version).Revision=$(revision) all: getdeps test getdeps: - @echo "Installing staticcheck" && go get -u honnef.co/go/tools/cmd/staticcheck && go install honnef.co/go/tools/cmd/staticcheck - @echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo/cmd/gocyclo && go install github.com/fzipp/gocyclo/cmd/gocyclo - @echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode && go install github.com/remyoudompheng/go-misc/deadcode - @echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell && go install github.com/client9/misspell/cmd/misspell - @echo "Installing golangci-lint" && go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 + @echo "Installing golangci-lint" && go get github.com/golangci/golangci-lint/cmd/golangci-lint && go install github.com/golangci/golangci-lint/cmd/golangci-lint go mod tidy - -verifiers: fmt lint cyclo deadcode spelling staticcheck - -fmt: - @echo "Running $@" - @gofmt -d . - lint: @echo "Running $@" - @${GOPATH}/bin/golangci-lint run - -cyclo: - @echo "Running $@" - @${GOPATH}/bin/gocyclo -over 100 . - -deadcode: - @echo "Running $@" - @${GOPATH}/bin/deadcode -test $(shell go list ./...) || true - -spelling: - @echo "Running $@" - @${GOPATH}/bin/misspell -i monitord -error `find .` - -staticcheck: - @echo "Running $@" - @${GOPATH}/bin/staticcheck -- ./... + golangci-lint run -c ../../.golangci.yml -test: verifiers +test: lint go test -v -vet=off ./... benchmarks: @@ -53,7 +26,7 @@ coverage: clean go test -v -vet=off ./... -coverprofile=coverage/coverage.out go tool cover -html=coverage/coverage.out -o coverage/coverage.html -testrace: verifiers +testrace: lint go test -v -race -vet=off ./... run: