From 890b847a36bda7c5ddfc0b0136bea4170c899f73 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 28 Oct 2025 12:53:10 +0100 Subject: [PATCH 01/27] kola: refactor qemu machine creation --- mantle/platform/machine/qemu/cluster.go | 263 +++++++++++++++--------- 1 file changed, 161 insertions(+), 102 deletions(-) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index a6b51e9219..cd5dcc9aa8 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -44,6 +44,13 @@ type Cluster struct { tearingDown bool } +type BuilderCallbacks struct { + BuilderInit func(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error + SetupDisks func(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error + SetupNetwork func(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error + OverrideDefaults func(builder *platform.QemuBuilder) error +} + func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) { return qc.NewMachineWithOptions(userdata, platform.MachineOptions{}) } @@ -59,6 +66,25 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options platform.QemuMachineOptions) (platform.Machine, error) { + return qc.NewMachineWithQemuOptionsAndBuilderCallbacks(userdata, options, BuilderCallbacks{ + BuilderInit: qc.InitDefaultBuilder, + SetupDisks: qc.SetupDefaultDisks, + SetupNetwork: qc.SetupDefaultNetwork, + OverrideDefaults: nil, + }) +} + +func (qc *Cluster) NewMachineWithQemuOptionsAndBuilderCallbacks(userdata *conf.UserData, options platform.QemuMachineOptions, callbacks BuilderCallbacks) (platform.Machine, error) { + if callbacks.BuilderInit == nil { + callbacks.BuilderInit = qc.InitDefaultBuilder + } + if callbacks.SetupDisks == nil { + callbacks.SetupDisks = qc.SetupDefaultDisks + } + if callbacks.SetupNetwork == nil { + callbacks.SetupNetwork = qc.SetupDefaultNetwork + } + id := uuid.New() dir := filepath.Join(qc.RuntimeConf().OutputDir, id) @@ -66,16 +92,15 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl return nil, err } - // hacky solution for cloud config ip substitution - // NOTE: escaping is not supported - qc.mu.Lock() + conf, err := qc.RenderUserDataForCloudIpSubstitution(userdata) + if err != nil { + return nil, err + } - conf, err := qc.RenderUserData(userdata, map[string]string{}) + confPath, err := qc.WriteIgnitionConfigToDir(conf, dir) if err != nil { - qc.mu.Unlock() return nil, err } - qc.mu.Unlock() journal, err := platform.NewJournal(dir) if err != nil { @@ -90,53 +115,133 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } builder := platform.NewQemuBuilder() - if options.DisablePDeathSig { - builder.Pdeathsig = false + defer builder.Close() + if err := callbacks.BuilderInit(options, builder); err != nil { + return nil, err + } + builder.UUID = qm.id + builder.ConsoleFile = qm.consolePath + builder.ConfigFile = confPath + // This one doesn't support configuring the path because we can't + // reliably change the Ignition config here... + for _, path := range qc.flight.opts.BindRO { + destpathrel := strings.TrimLeft(path, "/") + builder.MountHost(path, "/kola/host/"+destpathrel, true) } + if err := callbacks.SetupDisks(options, builder); err != nil { + return nil, err + } + if err := callbacks.SetupNetwork(options, builder); err != nil { + return nil, err + } + if callbacks.OverrideDefaults != nil { + if err := callbacks.OverrideDefaults(builder); err != nil { + return nil, err + } + } + // S390x specific stuff if qc.flight.opts.SecureExecution { if err := builder.SetSecureExecution(qc.flight.opts.SecureExecutionIgnitionPubKey, qc.flight.opts.SecureExecutionHostKey, conf); err != nil { return nil, err } } + if qc.flight.opts.Cex || options.Cex { + if err := builder.AddCexDevice(); err != nil { + return nil, err + } + } + + inst, err := builder.Exec() + if err != nil { + return nil, err + } + qm.inst = inst - var confPath string + err = util.Retry(6, 5*time.Second, func() error { + var err error + qm.ip, err = inst.SSHAddress() + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + + // Run StartMachine, which blocks on the machine being booted up enough + // for SSH access, but only if the caller didn't tell us not to. + if !options.SkipStartMachine { + if err := platform.StartMachine(qm, qm.journal); err != nil { + qm.Destroy() + return nil, err + } + } + + qc.AddMach(qm) + + // In this flow, nothing actually Wait()s for the QEMU process. Let's do it here + // and print something if it exited unexpectedly. Ideally in the future, this + // interface allows the test harness to provide e.g. a channel we can signal on so + // it knows to stop the test once QEMU dies. + go func() { + err := inst.Wait() + if err != nil && !qc.tearingDown { + plog.Errorf("QEMU process finished abnormally: %v", err) + } + }() + + return qm, nil +} + +func (qc *Cluster) Destroy() { + qc.tearingDown = true + qc.BaseCluster.Destroy() + qc.flight.DelCluster(qc) +} + +// hacky solution for cloud config ip substitution +// NOTE: escaping is not supported +func (qc *Cluster) RenderUserDataForCloudIpSubstitution(userdata *conf.UserData) (conf *conf.Conf, err error) { + qc.mu.Lock() + conf, err = qc.RenderUserData(userdata, map[string]string{}) + if err != nil { + qc.mu.Unlock() + return nil, err + } + qc.mu.Unlock() + return conf, nil +} + +func (qc *Cluster) WriteIgnitionConfigToDir(conf *conf.Conf, dir string) (confPath string, err error) { if conf.IsIgnition() { confPath = filepath.Join(dir, "ignition.json") - if err := conf.WriteFile(confPath); err != nil { - return nil, err + if err = conf.WriteFile(confPath); err != nil { + return confPath, err } - } else if conf.IsEmpty() { - } else { - return nil, fmt.Errorf("qemu only supports Ignition or empty configs") + } else if !conf.IsEmpty() { + return confPath, fmt.Errorf("qemu only supports Ignition or empty configs") } + return confPath, nil +} - builder.ConfigFile = confPath - defer builder.Close() - builder.UUID = qm.id +func (qc *Cluster) InitDefaultBuilder(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + if options.DisablePDeathSig { + builder.Pdeathsig = false + } if qc.flight.opts.Arch != "" { if err := builder.SetArchitecture(qc.flight.opts.Arch); err != nil { - return nil, err + return err } } if qc.flight.opts.Firmware != "" { builder.Firmware = qc.flight.opts.Firmware } - builder.Swtpm = qc.flight.opts.Swtpm - builder.Hostname = fmt.Sprintf("qemu%d", qc.BaseCluster.AllocateMachineSerial()) - builder.ConsoleFile = qm.consolePath - - // This one doesn't support configuring the path because we can't - // reliably change the Ignition config here... - for _, path := range qc.flight.opts.BindRO { - destpathrel := strings.TrimLeft(path, "/") - builder.MountHost(path, "/kola/host/"+destpathrel, true) - } - if qc.flight.opts.Memory != "" { memory, err := strconv.ParseInt(qc.flight.opts.Memory, 10, 32) if err != nil { - return nil, errors.Wrapf(err, "parsing memory option") + return errors.Wrapf(err, "parsing memory option") } builder.MemoryMiB = int(memory) } else if options.MinMemory != 0 { @@ -144,22 +249,30 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } else if qc.flight.opts.SecureExecution { builder.MemoryMiB = 4096 // SE needs at least 4GB } + builder.Swtpm = qc.flight.opts.Swtpm + builder.Hostname = fmt.Sprintf("qemu%d", qc.BaseCluster.AllocateMachineSerial()) + if options.Firmware != "" { + builder.Firmware = options.Firmware + } + if options.AppendKernelArgs != "" { + builder.AppendKernelArgs = options.AppendKernelArgs + } + if options.AppendFirstbootKernelArgs != "" { + builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs + } + return nil +} + +func (qc *Cluster) SetupDefaultDisks(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error { var primaryDisk platform.Disk if options.PrimaryDisk != "" { - var diskp *platform.Disk - if diskp, err = platform.ParseDisk(options.PrimaryDisk, true); err != nil { - return nil, errors.Wrapf(err, "parsing primary disk spec '%s'", options.PrimaryDisk) + diskp, err := platform.ParseDisk(options.PrimaryDisk, true) + if err != nil { + return errors.Wrapf(err, "parsing primary disk spec '%s'", options.PrimaryDisk) } primaryDisk = *diskp } - - if qc.flight.opts.Cex || options.Cex { - if err := builder.AddCexDevice(); err != nil { - return nil, err - } - } - if qc.flight.opts.Nvme || options.Nvme { primaryDisk.Channel = "nvme" } @@ -181,14 +294,16 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl if options.OverrideBackingFile != "" { primaryDisk.BackingFile = options.OverrideBackingFile } - - if err = builder.AddBootDisk(&primaryDisk); err != nil { - return nil, err + if err := builder.AddBootDisk(&primaryDisk); err != nil { + return err } - if err = builder.AddDisksFromSpecs(options.AdditionalDisks); err != nil { - return nil, err + if err := builder.AddDisksFromSpecs(options.AdditionalDisks); err != nil { + return err } + return nil +} +func (qc *Cluster) SetupDefaultNetwork(options platform.QemuMachineOptions, builder *platform.QemuBuilder) error { if len(options.HostForwardPorts) > 0 { builder.EnableUsermodeNetworking(options.HostForwardPorts, "") } else { @@ -200,64 +315,8 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl if options.AdditionalNics > 0 { builder.AddAdditionalNics(options.AdditionalNics) } - if options.AppendKernelArgs != "" { - builder.AppendKernelArgs = options.AppendKernelArgs - } - if options.AppendFirstbootKernelArgs != "" { - builder.AppendFirstbootKernelArgs = options.AppendFirstbootKernelArgs - } if !qc.RuntimeConf().InternetAccess { builder.RestrictNetworking = true } - if options.Firmware != "" { - builder.Firmware = options.Firmware - } - - inst, err := builder.Exec() - if err != nil { - return nil, err - } - qm.inst = inst - - err = util.Retry(6, 5*time.Second, func() error { - var err error - qm.ip, err = inst.SSHAddress() - if err != nil { - return err - } - return nil - }) - if err != nil { - return nil, err - } - - // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(qm, qm.journal); err != nil { - qm.Destroy() - return nil, err - } - } - - qc.AddMach(qm) - - // In this flow, nothing actually Wait()s for the QEMU process. Let's do it here - // and print something if it exited unexpectedly. Ideally in the future, this - // interface allows the test harness to provide e.g. a channel we can signal on so - // it knows to stop the test once QEMU dies. - go func() { - err := inst.Wait() - if err != nil && !qc.tearingDown { - plog.Errorf("QEMU process finished abnormally: %v", err) - } - }() - - return qm, nil -} - -func (qc *Cluster) Destroy() { - qc.tearingDown = true - qc.BaseCluster.Destroy() - qc.flight.DelCluster(qc) + return nil } From 53ceb11b79ff001230aed4565ba7bfaa04d82821 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 28 Oct 2025 12:57:32 +0100 Subject: [PATCH 02/27] kola: fold iso-live-login.* tests into 'kola run' --- mantle/cmd/kola/testiso.go | 40 --------- mantle/kola/registry/registry.go | 1 + mantle/kola/tests/iso/live-login.go | 132 ++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 40 deletions(-) create mode 100644 mantle/kola/tests/iso/live-login.go diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 199091e26b..9eff5feb41 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -82,10 +82,6 @@ var ( "iso-as-disk.uefi-secure", "iso-as-disk.4k.uefi", "iso-install.bios", - "iso-live-login.bios", - "iso-live-login.uefi", - "iso-live-login.uefi-secure", - "iso-live-login.4k.uefi", "iso-offline-install.bios", "iso-offline-install.mpath.bios", "iso-offline-install-fromram.4k.uefi", @@ -102,7 +98,6 @@ var ( "pxe-online-install.4k.uefi", } tests_s390x = []string{ - "iso-live-login.s390fw", "iso-offline-install.s390fw", "iso-offline-install.mpath.s390fw", "iso-offline-install.4k.s390fw", @@ -117,7 +112,6 @@ var ( //"iso-offline-install-iscsi.manual.s390fw", } tests_ppc64le = []string{ - "iso-live-login.ppcfw", "iso-offline-install.ppcfw", "iso-offline-install.mpath.ppcfw", "iso-offline-install-fromram.4k.ppcfw", @@ -133,8 +127,6 @@ var ( //"iso-offline-install-iscsi.manual.ppcfw", } tests_aarch64 = []string{ - "iso-live-login.uefi", - "iso-live-login.4k.uefi", "iso-offline-install.uefi", "iso-offline-install.mpath.uefi", "iso-offline-install-fromram.4k.uefi", @@ -631,8 +623,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { duration, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) case "iso-as-disk": duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) - case "iso-live-login": - duration, err = testLiveLogin(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) case "iso-install", "iso-offline-install", "iso-offline-install-fromram": @@ -1002,36 +992,6 @@ RequiredBy=fips-signal-ok.service return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } -func testLiveLogin(ctx context.Context, outdir string) (time.Duration, error) { - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("coreos.liveiso-success") - if err != nil { - return 0, err - } - - // No network device to test https://github.com/coreos/fedora-coreos-config/pull/326 - builder.Append("-net", "none") - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"coreos-liveiso-success"}) -} - func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { builddir := kola.CosaBuild.Dir isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/kola/registry/registry.go b/mantle/kola/registry/registry.go index c22c474e86..94a1ab66c3 100644 --- a/mantle/kola/registry/registry.go +++ b/mantle/kola/registry/registry.go @@ -7,6 +7,7 @@ import ( _ "github.com/coreos/coreos-assembler/mantle/kola/tests/etcd" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/fips" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ignition" + _ "github.com/coreos/coreos-assembler/mantle/kola/tests/iso" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/metadata" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/misc" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ostree" diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go new file mode 100644 index 0000000000..72ea6f071d --- /dev/null +++ b/mantle/kola/tests/iso/live-login.go @@ -0,0 +1,132 @@ +package testiso + +import ( + "bufio" + "fmt" + "io" + "path/filepath" + "strings" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/pkg/errors" + + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +func init() { + register.RegisterTest(®ister.Test{ + Run: isoLiveLogin, + ClusterSize: 0, + Name: "iso.live-login", + Description: "Verify ISO live login works.", + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + ExcludeArchitectures: []string{}, + }) + register.RegisterTest(®ister.Test{ + Run: isoLiveLoginUefi, + ClusterSize: 0, + Name: "iso.live-login.uefi", + Description: "Verify ISO live login works.", + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + ExcludeArchitectures: []string{"s390x", "ppcfw"}, + }) + register.RegisterTest(®ister.Test{ + Run: isoLiveLoginUefiSecure, + ClusterSize: 0, + Name: "iso.live-login.uefi-secure", + Description: "Verify ISO live login works.", + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + ExcludeArchitectures: []string{"s390x", "ppcfw"}, + }) +} + +func testLiveLogin(c cluster.TestCluster, enableUefi bool, enableUefiSecure bool) { + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) + } + + butane := conf.Butane(` +variant: fcos +version: 1.1.0`) + + errchan := make(chan error) + overrideFW := func(builder *platform.QemuBuilder) error { + switch { + case enableUefiSecure: + builder.Firmware = "uefi-secure" + case enableUefi: + builder.Firmware = "uefi" + } + return nil + } + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + // https://github.com/coreos/fedora-coreos-config/blob/testing-devel/overlay.d/05core/usr/lib/systemd/system/coreos-liveiso-success.service + output, err := builder.VirtioChannelRead("coreos.liveiso-success") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } + + // Read line in a goroutine and send errors to channel + go func() { + exp := "coreos-liveiso-success" + line, err := bufio.NewReader(output).ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line = strings.TrimSpace(line) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, exp) + return + } + // OK! + errchan <- nil + }() + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + return builder.AddIso(isopath, "", false) + } + + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err := pc.NewMachineWithQemuOptionsAndBuilderCallbacks(butane, platform.QemuMachineOptions{}, callacks) + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) + } + default: + c.Fatalf("Unsupported cluster type") + } + + err := <-errchan + if err != nil { + c.Fatal(err) + } +} + +func isoLiveLogin(c cluster.TestCluster) { + testLiveLogin(c, false, false) +} + +func isoLiveLoginUefi(c cluster.TestCluster) { + testLiveLogin(c, true, false) +} +func isoLiveLoginUefiSecure(c cluster.TestCluster) { + testLiveLogin(c, false, true) +} From 4bc38df6f50d1e719bb58e849284227454748d20 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 13 Nov 2025 15:07:56 +0100 Subject: [PATCH 03/27] kola: move metal.go to machine/qemu/ 'metal.go' is only used by testiso and contains code&data structures similar to those provided by 'machines/qemu/*.go'. --- mantle/cmd/kola/testiso.go | 11 ++++---- mantle/platform/{ => machine/qemu}/metal.go | 31 ++++++++------------- mantle/platform/qemu.go | 9 +++++- 3 files changed, 26 insertions(+), 25 deletions(-) rename mantle/platform/{ => machine/qemu}/metal.go (96%) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 9eff5feb41..2a94e71c0c 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -35,6 +35,7 @@ import ( "github.com/coreos/coreos-assembler/mantle/harness/reporters" "github.com/coreos/coreos-assembler/mantle/harness/testresult" "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" @@ -381,7 +382,7 @@ func getAllTests(build *util.LocalBuild) []string { } func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { - builder := platform.NewMetalQemuBuilderDefault() + builder := qemu.NewMetalQemuBuilderDefault() if enableUefiSecure { builder.Firmware = "uefi-secure" } else if enableUefi { @@ -548,7 +549,7 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } }() - baseInst := platform.Install{ + baseInst := qemu.Install{ CosaBuild: kola.CosaBuild, NmKeyfiles: make(map[string]string), } @@ -798,7 +799,7 @@ func printResult(test string, duration time.Duration, err error) bool { return false } -func testPXE(ctx context.Context, inst platform.Install, outdir string) (time.Duration, error) { +func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Duration, error) { if addNmKeyfile { return 0, errors.New("--add-nm-keyfile not yet supported for PXE") } @@ -862,7 +863,7 @@ func testPXE(ctx context.Context, inst platform.Install, outdir string) (time.Du return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) } -func testLiveIso(ctx context.Context, inst platform.Install, outdir string, minimal bool) (time.Duration, error) { +func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal bool) (time.Duration, error) { tmpd, err := os.MkdirTemp("", "kola-testiso") if err != nil { return 0, err @@ -1047,7 +1048,7 @@ func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { // 6 - /var/nested-ign.json contains an ignition config: // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes -func testLiveInstalliscsi(ctx context.Context, inst platform.Install, outdir string, butane string) (time.Duration, error) { +func testLiveInstalliscsi(ctx context.Context, inst qemu.Install, outdir string, butane string) (time.Duration, error) { builddir := kola.CosaBuild.Dir isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/platform/metal.go b/mantle/platform/machine/qemu/metal.go similarity index 96% rename from mantle/platform/metal.go rename to mantle/platform/machine/qemu/metal.go index 02a9da88c6..4c1dc4fec7 100644 --- a/mantle/platform/metal.go +++ b/mantle/platform/machine/qemu/metal.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package platform +package qemu import ( "bufio" @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/coreos/coreos-assembler/mantle/platform" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -45,14 +46,6 @@ const ( var baseKargs = []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} var ( - // TODO expose this as an API that can be used by cosa too - consoleKernelArgument = map[string]string{ - "x86_64": "ttyS0,115200n8", - "ppc64le": "hvc0", - "aarch64": "ttyAMA0", - "s390x": "ttysclp0", - } - bootStartedUnit = fmt.Sprintf(`[Unit] Description=TestISO Boot Started Requires=dev-virtio\\x2dports-bootstarted.device @@ -69,8 +62,8 @@ var ( // NewMetalQemuBuilderDefault returns a QEMU builder instance with some // defaults set up for bare metal. -func NewMetalQemuBuilderDefault() *QemuBuilder { - builder := NewQemuBuilder() +func NewMetalQemuBuilderDefault() *platform.QemuBuilder { + builder := platform.NewQemuBuilder() // https://github.com/coreos/fedora-coreos-tracker/issues/388 // https://github.com/coreos/fedora-coreos-docs/pull/46 builder.MemoryMiB = 4096 @@ -79,7 +72,7 @@ func NewMetalQemuBuilderDefault() *QemuBuilder { type Install struct { CosaBuild *util.LocalBuild - Builder *QemuBuilder + Builder *platform.QemuBuilder Insecure bool Native4k bool MultiPathDisk bool @@ -94,7 +87,7 @@ type Install struct { type InstalledMachine struct { Tempdir string - QemuInst *QemuInstance + QemuInst *platform.QemuInstance BootStartedErrorChannel chan error } @@ -123,7 +116,7 @@ func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offli } installerConfig := installerConfig{ - Console: []string{consoleKernelArgument[coreosarch.CurrentRpmArch()]}, + Console: []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]}, AppendKargs: renderCosaTestIsoDebugKargs(), } installerConfigData, err := yaml.Marshal(installerConfig) @@ -181,7 +174,7 @@ type pxeSetup struct { type installerRun struct { inst *Install - builder *QemuBuilder + builder *platform.QemuBuilder builddir string tempdir string @@ -348,7 +341,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) { } func renderBaseKargs() []string { - return append(baseKargs, fmt.Sprintf("console=%s", consoleKernelArgument[coreosarch.CurrentRpmArch()])) + return append(baseKargs, fmt.Sprintf("console=%s", platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()])) } func renderInstallKargs(t *installerRun, offline bool) []string { @@ -469,7 +462,7 @@ func (t *installerRun) completePxeSetup(kargs []string) error { return nil } -func switchBootOrderSignal(qinst *QemuInstance, bootstartedchan *os.File, booterrchan *chan error) { +func switchBootOrderSignal(qinst *platform.QemuInstance, bootstartedchan *os.File, booterrchan *chan error) { *booterrchan = make(chan error) go func() { err := qinst.Wait() @@ -529,7 +522,7 @@ func cat(outfile string, infiles ...string) error { return nil } -func (t *installerRun) run() (*QemuInstance, error) { +func (t *installerRun) run() (*platform.QemuInstance, error) { builder := t.builder netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", t.pxe.networkdevice) if t.pxe.bootindex == "" { @@ -626,7 +619,7 @@ func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgni // XXX: https://github.com/coreos/coreos-installer/issues/1171 if coreosarch.CurrentRpmArch() != "s390x" { - installerConfig.Console = []string{consoleKernelArgument[coreosarch.CurrentRpmArch()]} + installerConfig.Console = []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]} } if inst.MultiPathDisk { diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index c236601f30..07a15bd249 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -57,6 +57,13 @@ import ( var ( // ErrInitramfsEmergency is the marker error returned upon node blocking in emergency mode in initramfs. ErrInitramfsEmergency = errors.New("entered emergency.target in initramfs") + + ConsoleKernelArgument = map[string]string{ + "x86_64": "ttyS0,115200n8", + "ppc64le": "hvc0", + "aarch64": "ttyAMA0", + "s390x": "ttysclp0", + } ) // HostForwardPort contains details about port-forwarding for the VM. @@ -1531,7 +1538,7 @@ func (builder *QemuBuilder) setupIso() error { if kargsSupported, err := coreosInstallerSupportsISOKargs(); err != nil { return err } else if kargsSupported { - allargs := fmt.Sprintf("console=%s %s", consoleKernelArgument[coreosarch.CurrentRpmArch()], builder.AppendKernelArgs) + allargs := fmt.Sprintf("console=%s %s", ConsoleKernelArgument[coreosarch.CurrentRpmArch()], builder.AppendKernelArgs) instCmdKargs := exec.Command("coreos-installer", "iso", "kargs", "modify", "--append", allargs, isoEmbeddedPath) var stderrb bytes.Buffer instCmdKargs.Stderr = &stderrb From d6c5398e86f27e86d5d9f5e101b1ac8f10954f31 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 13 Nov 2025 16:20:57 +0100 Subject: [PATCH 04/27] kola: drop InstalledMachine struct --- mantle/cmd/kola/testiso.go | 12 +++-- mantle/platform/machine/qemu/machine.go | 61 +++++++++++++++++++------ mantle/platform/machine/qemu/metal.go | 39 +++++----------- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 2a94e71c0c..c649e0c9e5 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -855,12 +855,14 @@ func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Durati return 0, errors.Wrapf(err, "running PXE") } defer func() { - if err := mach.Destroy(); err != nil { + err := mach.DeleteTempdir() + mach.Destroy() + if err != nil { plog.Errorf("Failed to destroy PXE: %v", err) } }() - return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) + return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) } func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal bool) (time.Duration, error) { @@ -925,12 +927,14 @@ func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal return 0, errors.Wrapf(err, "running iso install") } defer func() { - if err := mach.Destroy(); err != nil { + err := mach.DeleteTempdir() + mach.Destroy() + if err != nil { plog.Errorf("Failed to destroy iso: %v", err) } }() - return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) + return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) } // testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system diff --git a/mantle/platform/machine/qemu/machine.go b/mantle/platform/machine/qemu/machine.go index 9ecfc62651..bb428838af 100644 --- a/mantle/platform/machine/qemu/machine.go +++ b/mantle/platform/machine/qemu/machine.go @@ -26,19 +26,29 @@ import ( ) type machine struct { - qc *Cluster - id string - inst *platform.QemuInstance - journal *platform.Journal - consolePath string - console string - ip string + qc *Cluster + id string + inst *platform.QemuInstance + journal *platform.Journal + consolePath string + console string + ip string + tempdir string + bootStartedErrorChannel chan error } func (m *machine) ID() string { return m.id } +func (m *machine) Instance() *platform.QemuInstance { + return m.inst +} + +func (m *machine) BootStartedErrorChannel() chan error { + return m.bootStartedErrorChannel +} + func (m *machine) IP() string { return m.ip } @@ -91,18 +101,41 @@ func (m *machine) WaitForSoftReboot(timeout time.Duration, oldSoftRebootsCount s return platform.WaitForMachineSoftReboot(m, m.journal, timeout, oldSoftRebootsCount) } +func (m *machine) DeleteTempdir() error { + var err error = nil + if m.tempdir != "" { + err = os.RemoveAll(m.tempdir) + m.tempdir = "" + } + return err +} + func (m *machine) Destroy() { - m.inst.Destroy() + if m.inst != nil { + m.inst.Destroy() + m.inst = nil + } + + if m.journal != nil { + m.journal.Destroy() + m.journal = nil + } - m.journal.Destroy() + if m.consolePath != "" { + if buf, err := os.ReadFile(m.consolePath); err == nil { + m.console = string(buf) + } else { + plog.Errorf("Error reading console for instance %v: %v", m.ID(), err) + } + } - if buf, err := os.ReadFile(m.consolePath); err == nil { - m.console = string(buf) - } else { - plog.Errorf("Error reading console for instance %v: %v", m.ID(), err) + if m.qc != nil { + m.qc.DelMach(m) } - m.qc.DelMach(m) + if err := m.DeleteTempdir(); err != nil { + plog.Errorf("Error removing tempdir for instance %v: %v", m.ID(), err) + } } func (m *machine) ConsoleOutput() string { diff --git a/mantle/platform/machine/qemu/metal.go b/mantle/platform/machine/qemu/metal.go index 4c1dc4fec7..d49e489bae 100644 --- a/mantle/platform/machine/qemu/metal.go +++ b/mantle/platform/machine/qemu/metal.go @@ -85,12 +85,6 @@ type Install struct { liveIgnition conf.Conf } -type InstalledMachine struct { - Tempdir string - QemuInst *platform.QemuInstance - BootStartedErrorChannel chan error -} - // Check that artifact has been built and locally exists func (inst *Install) checkArtifactsExist(artifacts []string) error { version := inst.CosaBuild.Meta.OstreeVersion @@ -109,7 +103,7 @@ func (inst *Install) checkArtifactsExist(artifacts []string) error { return nil } -func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offline bool) (*InstalledMachine, error) { +func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offline bool) (*machine, error) { artifacts := []string{"live-kernel", "live-rootfs"} if err := inst.checkArtifactsExist(artifacts); err != nil { return nil, err @@ -146,17 +140,6 @@ func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offli return mach, nil } -func (inst *InstalledMachine) Destroy() error { - if inst.QemuInst != nil { - inst.QemuInst.Destroy() - inst.QemuInst = nil - } - if inst.Tempdir != "" { - return os.RemoveAll(inst.Tempdir) - } - return nil -} - type kernelSetup struct { kernel, initramfs, rootfs string } @@ -544,7 +527,7 @@ func (t *installerRun) run() (*platform.QemuInstance, error) { return inst, nil } -func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*InstalledMachine, error) { +func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*machine, error) { t, err := inst.setup(kern) if err != nil { return nil, errors.Wrapf(err, "setting up install") @@ -572,11 +555,11 @@ func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*InstalledMachine, } tempdir := t.tempdir t.tempdir = "" // Transfer ownership - instmachine := InstalledMachine{ - QemuInst: qinst, - Tempdir: tempdir, + instmachine := machine{ + inst: qinst, + tempdir: tempdir, } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.BootStartedErrorChannel) + switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) return &instmachine, nil } @@ -592,7 +575,7 @@ type installerConfig struct { Console []string `yaml:"console,omitempty"` } -func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgnition conf.Conf, outdir string, offline, minimal bool) (*InstalledMachine, error) { +func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgnition conf.Conf, outdir string, offline, minimal bool) (*machine, error) { artifacts := []string{"live-iso"} if !offline { if inst.Native4k { @@ -845,10 +828,10 @@ After=dev-mapper-mpatha.device`) return nil, err } cleanupTempdir = false // Transfer ownership - instmachine := InstalledMachine{ - QemuInst: qinst, - Tempdir: tempdir, + instmachine := machine{ + inst: qinst, + tempdir: tempdir, } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.BootStartedErrorChannel) + switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) return &instmachine, nil } From c0f11977aa8c4a83c96ef15e75b4d7c5df0d70e8 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 14 Nov 2025 14:42:00 +0100 Subject: [PATCH 05/27] kola: move processing of ignition config (conf.UserData) to a separate function --- mantle/platform/machine/qemu/cluster.go | 33 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index cd5dcc9aa8..ad06978a39 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -74,7 +74,7 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl }) } -func (qc *Cluster) NewMachineWithQemuOptionsAndBuilderCallbacks(userdata *conf.UserData, options platform.QemuMachineOptions, callbacks BuilderCallbacks) (platform.Machine, error) { +func (qc *Cluster) NewMachineWithQemuOptionsAndBuilderCallbacks(userdata any, options platform.QemuMachineOptions, callbacks BuilderCallbacks) (platform.Machine, error) { if callbacks.BuilderInit == nil { callbacks.BuilderInit = qc.InitDefaultBuilder } @@ -92,12 +92,7 @@ func (qc *Cluster) NewMachineWithQemuOptionsAndBuilderCallbacks(userdata *conf.U return nil, err } - conf, err := qc.RenderUserDataForCloudIpSubstitution(userdata) - if err != nil { - return nil, err - } - - confPath, err := qc.WriteIgnitionConfigToDir(conf, dir) + conf, confPath, err := qc.ProcessIgnitionConfig(userdata, dir) if err != nil { return nil, err } @@ -201,6 +196,30 @@ func (qc *Cluster) Destroy() { qc.flight.DelCluster(qc) } +func (qc *Cluster) ProcessIgnitionConfig(cfg any, dir string) (*conf.Conf, string, error) { + var config *conf.Conf + var confPath string + var err error + switch p := cfg.(type) { + case *conf.UserData: + config, err = qc.RenderUserDataForCloudIpSubstitution(p) + if err != nil { + return nil, "", err + } + case *conf.Conf: + config = p + default: + return nil, "", fmt.Errorf("unknown config pointer type: %T", p) + } + if config != nil { + confPath, err = qc.WriteIgnitionConfigToDir(config, dir) + if err != nil { + return nil, "", err + } + } + return config, confPath, nil +} + // hacky solution for cloud config ip substitution // NOTE: escaping is not supported func (qc *Cluster) RenderUserDataForCloudIpSubstitution(userdata *conf.UserData) (conf *conf.Conf, err error) { From d06709a180f0f903b1b51336492e9fca1510e2b2 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 20 Nov 2025 13:47:40 +0100 Subject: [PATCH 06/27] kola: fold iso and miniso installation (offline, 4k, mpath, nm, uefi) tests --- mantle/cmd/kola/testiso.go | 211 -------- mantle/kola/tests/iso/live-iso.go | 811 ++++++++++++++++++++++++++++++ 2 files changed, 811 insertions(+), 211 deletions(-) create mode 100644 mantle/kola/tests/iso/live-iso.go diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index c649e0c9e5..32e37f7204 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -68,7 +68,6 @@ var ( enableUefi bool enableUefiSecure bool isOffline bool - isISOFromRAM bool // These tests only run on RHCOS tests_RHCOS_uefi = []string{ @@ -82,44 +81,23 @@ var ( "iso-as-disk.uefi", "iso-as-disk.uefi-secure", "iso-as-disk.4k.uefi", - "iso-install.bios", - "iso-offline-install.bios", - "iso-offline-install.mpath.bios", - "iso-offline-install-fromram.4k.uefi", "iso-offline-install-iscsi.ibft.uefi", "iso-offline-install-iscsi.ibft-with-mpath.bios", "iso-offline-install-iscsi.manual.bios", - "miniso-install.bios", - "miniso-install.nm.bios", - "miniso-install.4k.uefi", - "miniso-install.4k.nm.uefi", "pxe-offline-install.rootfs-appended.bios", "pxe-offline-install.4k.uefi", "pxe-online-install.bios", "pxe-online-install.4k.uefi", } tests_s390x = []string{ - "iso-offline-install.s390fw", - "iso-offline-install.mpath.s390fw", - "iso-offline-install.4k.s390fw", "pxe-online-install.rootfs-appended.s390fw", "pxe-offline-install.s390fw", - "miniso-install.s390fw", - "miniso-install.nm.s390fw", - "miniso-install.4k.nm.s390fw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.s390fw, //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", //"iso-offline-install-iscsi.manual.s390fw", } tests_ppc64le = []string{ - "iso-offline-install.ppcfw", - "iso-offline-install.mpath.ppcfw", - "iso-offline-install-fromram.4k.ppcfw", - "miniso-install.ppcfw", - "miniso-install.nm.ppcfw", - "miniso-install.4k.ppcfw", - "miniso-install.4k.nm.ppcfw", "pxe-online-install.rootfs-appended.ppcfw", "pxe-offline-install.4k.ppcfw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 @@ -128,13 +106,6 @@ var ( //"iso-offline-install-iscsi.manual.ppcfw", } tests_aarch64 = []string{ - "iso-offline-install.uefi", - "iso-offline-install.mpath.uefi", - "iso-offline-install-fromram.4k.uefi", - "miniso-install.uefi", - "miniso-install.nm.uefi", - "miniso-install.4k.uefi", - "miniso-install.4k.nm.uefi", "pxe-offline-install.uefi", "pxe-offline-install.rootfs-appended.4k.uefi", "pxe-online-install.uefi", @@ -148,8 +119,6 @@ var ( const ( installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" ) var liveOKSignal = "live-test-OK" @@ -227,18 +196,6 @@ ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' [Install] RequiredBy=multi-user.target` -var multipathedRoot = `[Unit] -Description=TestISO Verify Multipathed Root -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' -[Install] -RequiredBy=multi-user.target` - // This test is broken. Please fix! // https://github.com/coreos/coreos-assembler/issues/3554 var verifyNoEFIBootEntry = `[Unit] @@ -257,92 +214,6 @@ RequiredBy=coreos-installer.target # for iso-as-disk RequiredBy=multi-user.target` -// Verify that the volume ID is the OS name. See also -// https://github.com/openshift/assisted-image-service/pull/477. -// This is the same as the LABEL of the block device for ISO9660. See -// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 -var verifyIsoVolumeId = `[Unit] -Description=Verify ISO Volume ID -OnFailure=emergency.target -OnFailureJobMode=isolate -# only if we're actually mounting the ISO -ConditionPathIsMountPoint=/run/media/iso -[Service] -Type=oneshot -RemainAfterExit=yes -# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso -ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" -[Install] -RequiredBy=coreos-installer.target` - -// Unit to check that /run/media/iso is not mounted when -// coreos.liveiso.fromram kernel argument is passed -var isoNotMountedUnit = `[Unit] -Description=Verify ISO is not mounted when coreos.liveiso.fromram -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionKernelCommandLine=coreos.liveiso.fromram -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -RemainAfterExit=yes -# Would like to use SuccessExitStatus but it doesn't support what -# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 -ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" -[Install] -RequiredBy=coreos-installer.target` - -var nmConnectionId = "CoreOS DHCP" -var nmConnectionFile = "coreos-dhcp.nmconnection" -var nmConnection = fmt.Sprintf(`[connection] -id=%s -type=ethernet -# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will -# wait for a device to be present before exiting. See -# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 -wait-device-timeout=20000 - -[ipv4] -method=auto -`, nmConnectionId) - -var nmstateConfigFile = "/etc/nmstate/br-ex.yml" -var nmstateConfig = `interfaces: - - name: br-ex - type: linux-bridge - state: up - ipv4: - enabled: false - ipv6: - enabled: false - bridge: - port: [] -` - -// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. -var verifyNmKeyfile = fmt.Sprintf(`[Unit] -Description=TestISO Verify NM Keyfile Propagation -OnFailure=emergency.target -OnFailureJobMode=isolate -Wants=network-online.target -After=network-online.target -Before=live-signal-ok.service -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s -# Also verify nmstate config -ExecStart=/usr/bin/nmcli c show br-ex -[Install] -# for live system -RequiredBy=coreos-installer.target -# for target system -RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) - //go:embed resources/iscsi_butane_setup.yaml var iscsi_butane_config string @@ -612,12 +483,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { if kola.HasString("offline", strings.Split(components[0], "-")) { isOffline = true } - // For fromram it is a part of the first component. i.e. for - // iso-offline-install-fromram.uefi we need to search for 'fromram' in - // iso-offline-install-fromram, which is currently in components[0]. - if kola.HasString("fromram", strings.Split(components[0], "-")) { - isISOFromRAM = true - } switch components[0] { case "pxe-offline-install", "pxe-online-install": @@ -626,10 +491,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - case "iso-install", "iso-offline-install", "iso-offline-install-fromram": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) - case "miniso-install": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) case "iso-offline-install-iscsi": var butane_config string switch components[1] { @@ -865,78 +726,6 @@ func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Durati return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) } -func testLiveIso(ctx context.Context, inst qemu.Install, outdir string, minimal bool) (time.Duration, error) { - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, err - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - return 0, err - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(outdir) - if err != nil { - return 0, err - } - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - var isoKernelArgs []string - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) - liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - if inst.MultiPathDisk { - targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) - } - - if addNmKeyfile { - liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - // NM keyfile via `iso network embed` - inst.NmKeyfiles[nmConnectionFile] = nmConnection - // nmstate config via live Ignition config, propagated via - // --copy-network, which is enabled by inst.NmKeyfiles - liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) - } - - if isISOFromRAM { - isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg) - } - - mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, isOffline, minimal) - if err != nil { - return 0, errors.Wrapf(err, "running iso install") - } - defer func() { - err := mach.DeleteTempdir() - mach.Destroy() - if err != nil { - plog.Errorf("Failed to destroy iso: %v", err) - } - }() - - return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) -} - // testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system func testLiveFIPS(ctx context.Context, outdir string) (time.Duration, error) { tmpd, err := os.MkdirTemp("", "kola-testiso") diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go new file mode 100644 index 0000000000..ec22414068 --- /dev/null +++ b/mantle/kola/tests/iso/live-iso.go @@ -0,0 +1,811 @@ +package testiso + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/coreos/coreos-assembler/mantle/util" + + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" +) + +func init() { + register.RegisterTest(®ister.Test{ + Run: isoInstall, + ClusterSize: 0, + Name: "iso.live-install", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoOfflineInstall, + ClusterSize: 0, + Name: "iso.live-offline-install", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoOfflineInstall4k, + ClusterSize: 0, + Name: "iso.live-offline-install.4k", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"s390x"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoOfflineInstallMpath, + ClusterSize: 0, + Name: "iso.live-offline-install-mpath", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoOfflineInstallFromRam, + ClusterSize: 0, + Name: "iso.live-offline-install-fromram", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoOfflineInstallFromRam4k, + ClusterSize: 0, + Name: "iso.live-offline-install-fromram.4k", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"ppc64le"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoOfflineInstallFromRam4kUefi, + ClusterSize: 0, + Name: "iso.live-offline-install-fromram.4k.uefi", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoMinisoInstall, + ClusterSize: 0, + Name: "iso.miniso-install", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoMinisoInstall4k, + ClusterSize: 0, + Name: "iso.miniso-install.4k", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"ppc64le"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoMinisoInstall4kUefi, + ClusterSize: 0, + Name: "iso.miniso-install.4k.uefi", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoMinisoInstallNm, + ClusterSize: 0, + Name: "iso.miniso-install.nm", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoMinisoInstall4kNm, + ClusterSize: 0, + Name: "iso.miniso-install.4k.nm", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"ppc64le", "s390x"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoMinisoInstall4kNmUefi, + ClusterSize: 0, + Name: "iso.miniso-install.4k.nm.uefi", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"ppc64le", "s390x"}, + }) +} + +var liveOKSignal = "live-test-OK" +var liveSignalOKUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Live ISO Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target`, liveOKSignal) + +var signalCompleteString = "coreos-installer-test-OK" +var signalCompletionUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=multi-user.target`, signalCompleteString) + +var signalEmergencyString = "coreos-installer-test-entered-emergency-target" +var signalFailureUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Failure +Requires=dev-virtio\\x2dports-testisocompletion.device +DefaultDependencies=false +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=emergency.target`, signalEmergencyString) + +var multipathedRoot = `[Unit] +Description=TestISO Verify Multipathed Root +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' +[Install] +RequiredBy=multi-user.target` + +var checkNoIgnition = ` +[Unit] +Description=TestISO Verify No Ignition Config +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +After=coreos-ignition-firstboot-complete.service +RequiresMountsFor=/boot +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' +[Install] +RequiredBy=multi-user.target` + +// This test is broken. Please fix! +// https://github.com/coreos/coreos-assembler/issues/3554 +var verifyNoEFIBootEntry = ` +[Unit] +Description=TestISO Verify No EFI Boot Entry +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionPathExists=/sys/firmware/efi +Before=live-signal-ok.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target` + +// Verify that the volume ID is the OS name. See also +// https://github.com/openshift/assisted-image-service/pull/477. +// This is the same as the LABEL of the block device for ISO9660. See +// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 +var verifyIsoVolumeId = ` +[Unit] +Description=Verify ISO Volume ID +OnFailure=emergency.target +OnFailureJobMode=isolate +# only if we're actually mounting the ISO +ConditionPathIsMountPoint=/run/media/iso +[Service] +Type=oneshot +RemainAfterExit=yes +# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso +ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" +[Install] +RequiredBy=coreos-installer.target` + +// Unit to check that /run/media/iso is not mounted when +// coreos.liveiso.fromram kernel argument is passed +var isoNotMountedUnit = ` +[Unit] +Description=Verify ISO is not mounted when coreos.liveiso.fromram +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionKernelCommandLine=coreos.liveiso.fromram +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +RemainAfterExit=yes +# Would like to use SuccessExitStatus but it doesn't support what +# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 +ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" +[Install] +RequiredBy=coreos-installer.target` + +var nmConnectionId = "CoreOS DHCP" +var nmConnectionFile = "coreos-dhcp.nmconnection" +var nmConnection = fmt.Sprintf(`[connection] +id=%s +type=ethernet +# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will +# wait for a device to be present before exiting. See +# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 +wait-device-timeout=20000 + +[ipv4] +method=auto +`, nmConnectionId) + +var nmstateConfigFile = "/etc/nmstate/br-ex.yml" +var nmstateConfig = `interfaces: + - name: br-ex + type: linux-bridge + state: up + ipv4: + enabled: false + ipv6: + enabled: false + bridge: + port: [] +` + +// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. +var verifyNmKeyfile = fmt.Sprintf(`[Unit] +Description=TestISO Verify NM Keyfile Propagation +OnFailure=emergency.target +OnFailureJobMode=isolate +Wants=network-online.target +After=network-online.target +Before=live-signal-ok.service +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s +# Also verify nmstate config +ExecStart=/usr/bin/nmcli c show br-ex +[Install] +# for live system +RequiredBy=coreos-installer.target +# for target system +RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) + +type IsoTestOpts struct { + // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") + instInsecure bool + // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") + console bool + addNmKeyfile bool + enable4k bool + enableMultipath bool + isOffline bool + isISOFromRAM bool + isMiniso bool + enableUefi bool + enableUefiSecure bool +} + +func (o *IsoTestOpts) SetInsecureOnDevBuild() { + // Ignore signing verification by default when running with development build + // https://github.com/coreos/fedora-coreos-tracker/issues/908 + if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + o.instInsecure = true + fmt.Printf("Detected development build; disabling signature verification\n") + } +} + +const ( + installTimeoutMins = 12 + // https://github.com/coreos/fedora-coreos-config/pull/2544 + liveISOFromRAMKarg = "coreos.liveiso.fromram" +) + +func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { + builder := qemu.NewMetalQemuBuilderDefault() + if opts.enableUefiSecure { + builder.Firmware = "uefi-secure" + } else if opts.enableUefi { + builder.Firmware = "uefi" + } + + if err := os.MkdirAll(outdir, 0755); err != nil { + return nil, err + } + + builder.InheritConsole = opts.console + if !opts.console { + builder.ConsoleFile = filepath.Join(outdir, "console.txt") + } + + if kola.QEMUOptions.Memory != "" { + parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) + if err != nil { + return nil, err + } + builder.MemoryMiB = int(parsedMem) + } + + return builder, nil +} + +func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + return nil, nil, err + } + + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return nil, nil, err + } + + err = forwardJournal(outdir, builder, config) + if err != nil { + return nil, nil, err + } + + return builder, config, nil +} + +func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { + journalPipe, err := builder.VirtioJournal(config, "") + if err != nil { + return err + } + journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + + go func() { + _, err := io.Copy(journalOut, journalPipe) + if err != nil && err != io.EOF { + panic(err) + } + }() + + return nil +} + +func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, config, err := newQemuBuilder(opts, outdir) + + if err != nil { + return nil, nil, err + } + + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return nil, nil, err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return nil, nil, err + } + } + + return builder, config, nil +} + +func isoInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoOfflineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoOfflineInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoOfflineInstallMpath(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableMultipath: true, + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + isOffline: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoOfflineInstallFromRam(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + isOffline: true, + isISOFromRAM: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoOfflineInstallFromRam4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + isOffline: true, + isISOFromRAM: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + isOffline: true, + isISOFromRAM: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoMinisoInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoMinisoInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + isMiniso: true, + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoMinisoInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoMinisoInstallNm(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoMinisoInstall4kNm(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + enable4k: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoMinisoInstall4kNmUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + enable4k: true, + enableUefi: true, + isMiniso: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + } + + inst := qemu.Install{ + CosaBuild: kola.CosaBuild, + NmKeyfiles: make(map[string]string), + Insecure: opts.instInsecure, + Native4k: opts.enable4k, + MultiPathDisk: opts.enableMultipath, + } + + tmpd, err := os.MkdirTemp("", "kola-iso.live") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpd) + + sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + if err != nil { + c.Fatal(err) + } + + builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + if err != nil { + c.Fatal(err) + } + inst.Builder = builder + completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + var isoKernelArgs []string + var keys []string + keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) + virtioJournalConfig.AddAuthorizedKeys("core", keys) + + liveConfig := *virtioJournalConfig + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) + liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) + + targetConfig := *virtioJournalConfig + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + if inst.MultiPathDisk { + targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) + } + + if opts.addNmKeyfile { + liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) + // NM keyfile via `iso network embed` + inst.NmKeyfiles[nmConnectionFile] = nmConnection + // nmstate config via live Ignition config, propagated via + // --copy-network, which is enabled by inst.NmKeyfiles + liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) + } + + if opts.isISOFromRAM { + isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg) + } + + mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, opts.isOffline, opts.isMiniso) + if err != nil { + c.Fatal(err) + } + qc.AddMach(mach) + err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + if err != nil { + c.Fatal(err) + } +} + +func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { + ctx := c.Context() + + errchan := make(chan error) + go func() { + timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 + time.Sleep(timeout) + errchan <- fmt.Errorf("timed out after %v", timeout) + }() + if !console { + go func() { + errBuf, err := inst.WaitIgnitionError(ctx) + if err == nil { + if errBuf != "" { + c.Logf("entered emergency.target in initramfs") + path := filepath.Join(outdir, "ignition-virtio-dump.txt") + if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { + c.Errorf("Failed to write journal: %v", err) + } + err = platform.ErrInitramfsEmergency + } + } + if err != nil { + errchan <- err + } + }() + } + go func() { + err := inst.Wait() + // only one Wait() gets process data, so also manually check for signal + //plog.Debugf("qemu exited err=%v", err) + if err == nil && inst.Signaled() { + err = errors.New("process killed") + } + if err != nil { + errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") + } + time.Sleep(1 * time.Minute) + errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") + }() + go func() { + r := bufio.NewReader(qchan) + for _, exp := range expected { + l, err := r.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line := strings.TrimSpace(l) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) + return + } + } + // OK! + errchan <- nil + }() + go func() { + //check for error when switching boot order + if booterrchan != nil { + if err := <-booterrchan; err != nil { + errchan <- err + } + } + }() + err := <-errchan + if err == nil { + // No error so far, check the console and journal files + consoleFile := filepath.Join(outdir, "console.txt") + journalFile := filepath.Join(outdir, "journal.txt") + files := []string{consoleFile, journalFile} + for _, file := range files { + fileName := filepath.Base(file) + // Check if the file exists + _, err := os.Stat(file) + if os.IsNotExist(err) { + fmt.Printf("The file: %v does not exist\n", fileName) + continue + } else if err != nil { + fmt.Println(err) + return err + } + // Read the contents of the file + fileContent, err := os.ReadFile(file) + if err != nil { + fmt.Println(err) + return err + } + // Check for badness with CheckConsole + warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) + if len(badlines) > 0 { + for _, badline := range badlines { + if warnOnly { + c.Errorf("bad log line detected: %v", badline) + } else { + c.Logf("bad log line detected: %v", badline) + } + } + if !warnOnly { + err = fmt.Errorf("errors found in log files") + return err + } + } + } + } + return err +} From c7d96645ef6f5460932be02ed945d16d25f273f9 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 20 Nov 2025 16:32:09 +0100 Subject: [PATCH 07/27] kola: fold PXE installation (offline, 4k, uefi) tests --- mantle/cmd/kola/testiso.go | 162 ---------------- mantle/kola/tests/iso/live-iso.go | 296 +++++++++++++++++++++++++++++- 2 files changed, 295 insertions(+), 163 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 32e37f7204..ee8219341f 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -62,7 +62,6 @@ var ( console bool - addNmKeyfile bool enable4k bool enableMultipath bool enableUefi bool @@ -84,32 +83,20 @@ var ( "iso-offline-install-iscsi.ibft.uefi", "iso-offline-install-iscsi.ibft-with-mpath.bios", "iso-offline-install-iscsi.manual.bios", - "pxe-offline-install.rootfs-appended.bios", - "pxe-offline-install.4k.uefi", - "pxe-online-install.bios", - "pxe-online-install.4k.uefi", } tests_s390x = []string{ - "pxe-online-install.rootfs-appended.s390fw", - "pxe-offline-install.s390fw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.s390fw, //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", //"iso-offline-install-iscsi.manual.s390fw", } tests_ppc64le = []string{ - "pxe-online-install.rootfs-appended.ppcfw", - "pxe-offline-install.4k.ppcfw", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.ppcfw", //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", //"iso-offline-install-iscsi.manual.ppcfw", } tests_aarch64 = []string{ - "pxe-offline-install.uefi", - "pxe-offline-install.rootfs-appended.4k.uefi", - "pxe-online-install.uefi", - "pxe-online-install.4k.uefi", // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 //"iso-offline-install-iscsi.ibft.uefi", //"iso-offline-install-iscsi.ibft-with-mpath.uefi", @@ -139,36 +126,6 @@ RequiredBy=coreos-installer.target RequiredBy=multi-user.target `, liveOKSignal) -var downloadCheck = `[Unit] -Description=TestISO Verify CoreOS Installer Download -After=coreos-installer.service -Before=coreos-installer.target -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" -ExecStart=/bin/sh -c "/usr/bin/udevadm settle" -ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" -ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" -[Install] -RequiredBy=coreos-installer.target -` - -var signalCompleteString = "coreos-installer-test-OK" -var signalCompletionUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=multi-user.target -`, signalCompleteString) - var signalEmergencyString = "coreos-installer-test-entered-emergency-target" var signalFailureUnit = fmt.Sprintf(`[Unit] Description=TestISO Signal Failure @@ -182,20 +139,6 @@ ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && s RequiredBy=emergency.target `, signalEmergencyString) -var checkNoIgnition = `[Unit] -Description=TestISO Verify No Ignition Config -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -After=coreos-ignition-firstboot-complete.service -RequiresMountsFor=/boot -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' -[Install] -RequiredBy=multi-user.target` - // This test is broken. Please fix! // https://github.com/coreos/coreos-assembler/issues/3554 var verifyNoEFIBootEntry = `[Unit] @@ -319,39 +262,6 @@ func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.C return nil } -func newQemuBuilderWithDisk(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, config, err := newQemuBuilder(outdir) - - if err != nil { - return nil, nil, err - } - - sectorSize := 0 - if enable4k { - sectorSize = 4096 - } - - disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - MultiPathDisk: enableMultipath, - } - - //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup - if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { - // s390x and aarch64 need to use bootindex as they don't support boot once - if err := builder.AddDisk(&disk); err != nil { - return nil, nil, err - } - } else { - if err := builder.AddPrimaryDisk(&disk); err != nil { - return nil, nil, err - } - } - - return builder, config, nil -} - // See similar semantics in the `filterTests` of `kola.go`. func filterTests(tests []string, patterns []string) ([]string, error) { r := []string{} @@ -448,7 +358,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { return err } - addNmKeyfile = false enable4k = false enableMultipath = false enableUefi = false @@ -465,9 +374,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { enable4k = true inst.Native4k = true } - if kola.HasString("nm", components) { - addNmKeyfile = true - } if kola.HasString("mpath", components) { enableMultipath = true inst.MultiPathDisk = true @@ -485,8 +391,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } switch components[0] { - case "pxe-offline-install", "pxe-online-install": - duration, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) case "iso-as-disk": duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": @@ -660,72 +564,6 @@ func printResult(test string, duration time.Duration, err error) bool { return false } -func testPXE(ctx context.Context, inst qemu.Install, outdir string) (time.Duration, error) { - if addNmKeyfile { - return 0, errors.New("--add-nm-keyfile not yet supported for PXE") - } - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, errors.Wrapf(err, "creating tempdir") - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - return 0, errors.Wrapf(err, "creating SSH AuthorizedKey") - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(outdir) - if err != nil { - return 0, errors.Wrapf(err, "creating QemuBuilder") - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, errors.Wrapf(err, "setting up virtio-serial channel") - } - - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - if isOffline { - contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) - liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) - } - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - - mach, err := inst.PXE(pxeKernelArgs, liveConfig, targetConfig, isOffline) - if err != nil { - return 0, errors.Wrapf(err, "running PXE") - } - defer func() { - err := mach.DeleteTempdir() - mach.Destroy() - if err != nil { - plog.Errorf("Failed to destroy PXE: %v", err) - } - }() - - return awaitCompletion(ctx, mach.Instance(), outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) -} - // testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system func testLiveFIPS(ctx context.Context, outdir string) (time.Duration, error) { tmpd, err := os.MkdirTemp("", "kola-testiso") diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index ec22414068..6ef2634e76 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -101,6 +101,7 @@ func init() { Architectures: []string{"x86_64", "aarch64"}, }) + // Miniso register.RegisterTest(®ister.Test{ Run: isoMinisoInstall, ClusterSize: 0, @@ -166,6 +167,106 @@ func init() { Platforms: []string{"qemu"}, Architectures: []string{"ppc64le", "s390x"}, }) + + // PXE + register.RegisterTest(®ister.Test{ + Run: isoPxeOnlineInstall, + ClusterSize: 0, + Name: "iso.pxe-online-install", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOnlineInstall4kUefi, + ClusterSize: 0, + Name: "iso.pxe-online-install.4k.uefi", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOnlineInstallRootfsAppended, + ClusterSize: 0, + Name: "iso.pxe-online-install.rootfs-appended", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "ppc64le", "s390x"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOfflineInstall, + ClusterSize: 0, + Name: "iso.pxe-offline-install", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOfflineInstallUefi, + ClusterSize: 0, + Name: "iso.pxe-offline-install.uefi", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOfflineInstall4k, + ClusterSize: 0, + Name: "iso.pxe-offline-install.4k", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"ppc64le", "s390x"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOfflineInstall4kUefi, + ClusterSize: 0, + Name: "iso.pxe-offline-install.4k.uefi", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOfflineInstallRootfsAppended, + ClusterSize: 0, + Name: "iso.pxe-offline-install.rootfs-appended", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{}, + }) + + register.RegisterTest(®ister.Test{ + Run: isoPxeOfflineInstallRootfsAppended4k, + ClusterSize: 0, + Name: "iso.pxe-offline-install.rootfs-appended.4k", + Description: "Verify ISO live login works.", + Timeout: 12 * time.Minute, + Flags: []register.Flag{}, + Platforms: []string{"qemu"}, + Architectures: []string{"aarch64"}, + }) } var liveOKSignal = "live-test-OK" @@ -350,6 +451,8 @@ RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") instInsecure bool + // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") + pxeKernelArgs []string // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") console bool addNmKeyfile bool @@ -360,6 +463,7 @@ type IsoTestOpts struct { isMiniso bool enableUefi bool enableUefiSecure bool + pxeAppendRootfs bool } func (o *IsoTestOpts) SetInsecureOnDevBuild() { @@ -367,7 +471,7 @@ func (o *IsoTestOpts) SetInsecureOnDevBuild() { // https://github.com/coreos/fedora-coreos-tracker/issues/908 if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { o.instInsecure = true - fmt.Printf("Detected development build; disabling signature verification\n") + //fmt.Printf("Detected development build; disabling signature verification\n") } } @@ -402,6 +506,13 @@ func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, builder.MemoryMiB = int(parsedMem) } + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + return builder, nil } @@ -604,6 +715,89 @@ func isoMinisoInstall4kNmUefi(c cluster.TestCluster) { isoLiveIso(c, opts) } +func isoPxeOnlineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstallRootfsAppended(c cluster.TestCluster) { + opts := IsoTestOpts{ + pxeAppendRootfs: true, + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enable4k: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallRootfsAppended(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + pxeAppendRootfs: true, + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallRootfsAppended4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + pxeAppendRootfs: true, + enable4k: true, + enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { var outdir string var qc *qemu.Cluster @@ -694,6 +888,106 @@ func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { } } +var downloadCheck = `[Unit] +Description=TestISO Verify CoreOS Installer Download +After=coreos-installer.service +Before=coreos-installer.target +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" +ExecStart=/bin/sh -c "/usr/bin/udevadm settle" +ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" +ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" +[Install] +RequiredBy=coreos-installer.target +` + +func testPXE(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + var qc *qemu.Cluster + + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + if opts.addNmKeyfile { + c.Fatal("--add-nm-keyfile not yet supported for PXE") + } + + inst := qemu.Install{ + CosaBuild: kola.CosaBuild, + NmKeyfiles: make(map[string]string), + Insecure: opts.instInsecure, + Native4k: opts.enable4k, + MultiPathDisk: opts.enableMultipath, + PxeAppendRootfs: opts.pxeAppendRootfs, + } + + tmpd, err := os.MkdirTemp("", "kola-iso.pxe") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpd) + + sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + if err != nil { + c.Fatal(err) + } + + builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + if err != nil { + c.Fatal(err) + } + + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + + inst.Builder = builder + completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) // , "setting up virtio-serial channel") + } + + var keys []string + keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) + virtioJournalConfig.AddAuthorizedKeys("core", keys) + + liveConfig := *virtioJournalConfig + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + if opts.isOffline { + contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) + liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) + } + + targetConfig := *virtioJournalConfig + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + + mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + if err != nil { + c.Fatal(err) + } + qc.AddMach(mach) + + err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + if err != nil { + c.Fatal(err) + } +} + func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { ctx := c.Context() From 1965e18f0ec02f4ab3dafaab11f4675dbc3988ec Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 09:43:37 +0100 Subject: [PATCH 08/27] kola: remove duplicated code in iso.* tests --- mantle/kola/tests/iso/live-iso.go | 345 +++++++--------------------- mantle/kola/tests/iso/live-login.go | 30 +-- 2 files changed, 88 insertions(+), 287 deletions(-) diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 6ef2634e76..ab92cdc15c 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -23,250 +23,56 @@ import ( "github.com/coreos/coreos-assembler/mantle/platform/conf" ) -func init() { - register.RegisterTest(®ister.Test{ - Run: isoInstall, - ClusterSize: 0, - Name: "iso.live-install", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64"}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoOfflineInstall, - ClusterSize: 0, - Name: "iso.live-offline-install", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoOfflineInstall4k, - ClusterSize: 0, - Name: "iso.live-offline-install.4k", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"s390x"}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoOfflineInstallMpath, - ClusterSize: 0, - Name: "iso.live-offline-install-mpath", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoOfflineInstallFromRam, - ClusterSize: 0, - Name: "iso.live-offline-install-fromram", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64"}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoOfflineInstallFromRam4k, - ClusterSize: 0, - Name: "iso.live-offline-install-fromram.4k", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"ppc64le"}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoOfflineInstallFromRam4kUefi, - ClusterSize: 0, - Name: "iso.live-offline-install-fromram.4k.uefi", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64", "aarch64"}, - }) - - // Miniso - register.RegisterTest(®ister.Test{ - Run: isoMinisoInstall, - ClusterSize: 0, - Name: "iso.miniso-install", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoMinisoInstall4k, - ClusterSize: 0, - Name: "iso.miniso-install.4k", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"ppc64le"}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoMinisoInstall4kUefi, - ClusterSize: 0, - Name: "iso.miniso-install.4k.uefi", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64", "aarch64"}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoMinisoInstallNm, - ClusterSize: 0, - Name: "iso.miniso-install.nm", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoMinisoInstall4kNm, - ClusterSize: 0, - Name: "iso.miniso-install.4k.nm", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"ppc64le", "s390x"}, - }) +const ( + installTimeoutMins = 12 + // https://github.com/coreos/fedora-coreos-config/pull/2544 + liveISOFromRAMKarg = "coreos.liveiso.fromram" +) - register.RegisterTest(®ister.Test{ - Run: isoMinisoInstall4kNmUefi, +func isoTest(name string, run func(c cluster.TestCluster), arch []string) *register.Test { + return ®ister.Test{ + Run: run, ClusterSize: 0, - Name: "iso.miniso-install.4k.nm.uefi", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, + Name: "iso." + name, + Timeout: installTimeoutMins * time.Minute, Platforms: []string{"qemu"}, - Architectures: []string{"ppc64le", "s390x"}, - }) + Architectures: arch, + } +} - // PXE - register.RegisterTest(®ister.Test{ - Run: isoPxeOnlineInstall, - ClusterSize: 0, - Name: "iso.pxe-online-install", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64", "aarch64"}, - }) +func init() { + register.RegisterTest(isoTest("install", isoInstall, []string{"x86_64"})) - register.RegisterTest(®ister.Test{ - Run: isoPxeOnlineInstall4kUefi, - ClusterSize: 0, - Name: "iso.pxe-online-install.4k.uefi", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64", "aarch64"}, - }) + register.RegisterTest(isoTest("offline-install", isoOfflineInstall, []string{"x86_64", "s390x", "ppc64le"})) + register.RegisterTest(isoTest("offline-install.uefi", isoOfflineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("offline-install.4k", isoOfflineInstall4k, []string{"s390x"})) + register.RegisterTest(isoTest("offline-install.mpath", isoOfflineInstallMpath, []string{"x86_64", "s390x", "ppc64le"})) + register.RegisterTest(isoTest("offline-install.mpath.uefi", isoOfflineInstallMpathUefi, []string{"aarch64"})) - register.RegisterTest(®ister.Test{ - Run: isoPxeOnlineInstallRootfsAppended, - ClusterSize: 0, - Name: "iso.pxe-online-install.rootfs-appended", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64", "ppc64le", "s390x"}, - }) + register.RegisterTest(isoTest("offline-install-fromram.4k", isoOfflineInstallFromRam4k, []string{"ppc64le"})) + register.RegisterTest(isoTest("offline-install-fromram.4k.uefi", isoOfflineInstallFromRam4kUefi, []string{"x86_64", "aarch64"})) - register.RegisterTest(®ister.Test{ - Run: isoPxeOfflineInstall, - ClusterSize: 0, - Name: "iso.pxe-offline-install", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64"}, - }) + register.RegisterTest(isoTest("miniso-install", isoMinisoInstall, []string{"x86_64", "s390x", "ppc64le"})) + register.RegisterTest(isoTest("miniso-install.uefi", isoMinisoInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("miniso-install.4k", isoMinisoInstall4k, []string{"ppc64le"})) + register.RegisterTest(isoTest("miniso-install.4k.uefi", isoMinisoInstall4kUefi, []string{"x86_64", "aarch64"})) - register.RegisterTest(®ister.Test{ - Run: isoPxeOfflineInstallUefi, - ClusterSize: 0, - Name: "iso.pxe-offline-install.uefi", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64", "aarch64"}, - }) + register.RegisterTest(isoTest("miniso-install.nm", isoMinisoInstallNm, []string{"x86_64", "s390x", "ppc64le"})) + register.RegisterTest(isoTest("miniso-install.nm.uefi", isoMinisoInstallNmUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("miniso-install.4k.nm", isoMinisoInstall4kNm, []string{"ppc64le", "s390x"})) + register.RegisterTest(isoTest("miniso-install.4k.nm.uefi", isoMinisoInstall4kNmUefi, []string{"x86_64", "aarch64"})) - register.RegisterTest(®ister.Test{ - Run: isoPxeOfflineInstall4k, - ClusterSize: 0, - Name: "iso.pxe-offline-install.4k", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"ppc64le", "s390x"}, - }) + register.RegisterTest(isoTest("pxe-online-install", isoPxeOnlineInstall, []string{"x86_64"})) + register.RegisterTest(isoTest("pxe-online-install.uefi", isoPxeOnlineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("pxe-online-install.4k.uefi", isoPxeOnlineInstall4kUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("pxe-online-install.rootfs-appended", isoPxeOnlineInstallRootfsAppended, []string{"ppc64le", "s390x"})) - register.RegisterTest(®ister.Test{ - Run: isoPxeOfflineInstall4kUefi, - ClusterSize: 0, - Name: "iso.pxe-offline-install.4k.uefi", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"x86_64", "aarch64"}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoPxeOfflineInstallRootfsAppended, - ClusterSize: 0, - Name: "iso.pxe-offline-install.rootfs-appended", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{}, - }) - - register.RegisterTest(®ister.Test{ - Run: isoPxeOfflineInstallRootfsAppended4k, - ClusterSize: 0, - Name: "iso.pxe-offline-install.rootfs-appended.4k", - Description: "Verify ISO live login works.", - Timeout: 12 * time.Minute, - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - Architectures: []string{"aarch64"}, - }) + register.RegisterTest(isoTest("pxe-offline-install", isoPxeOfflineInstall, []string{"s390x"})) + register.RegisterTest(isoTest("pxe-offline-install.uefi", isoPxeOfflineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("pxe-offline-install.4k", isoPxeOfflineInstall4k, []string{"ppc64le"})) + register.RegisterTest(isoTest("pxe-offline-install.4k.uefi", isoPxeOfflineInstall4kUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended", isoPxeOfflineInstallRootfsAppended, []string{"x86_64"})) + register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended.4k.uefi", isoPxeOfflineInstallRootfsAppended4kUefi, []string{"aarch64"})) } var liveOKSignal = "live-test-OK" @@ -475,12 +281,6 @@ func (o *IsoTestOpts) SetInsecureOnDevBuild() { } } -const ( - installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" -) - func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { builder := qemu.NewMetalQemuBuilderDefault() if opts.enableUefiSecure { @@ -589,16 +389,22 @@ func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuil } func isoInstall(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoOfflineInstall(c cluster.TestCluster) { opts := IsoTestOpts{ - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + isOffline: true, } opts.SetInsecureOnDevBuild() isoLiveIso(c, opts) } -func isoOfflineInstall(c cluster.TestCluster) { +func isoOfflineInstallUefi(c cluster.TestCluster) { opts := IsoTestOpts{ - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + enableUefi: true, isOffline: true, } opts.SetInsecureOnDevBuild() @@ -607,9 +413,8 @@ func isoOfflineInstall(c cluster.TestCluster) { func isoOfflineInstall4k(c cluster.TestCluster) { opts := IsoTestOpts{ - enable4k: true, - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", - isOffline: true, + enable4k: true, + isOffline: true, } opts.SetInsecureOnDevBuild() isoLiveIso(c, opts) @@ -618,18 +423,17 @@ func isoOfflineInstall4k(c cluster.TestCluster) { func isoOfflineInstallMpath(c cluster.TestCluster) { opts := IsoTestOpts{ enableMultipath: true, - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", isOffline: true, } opts.SetInsecureOnDevBuild() isoLiveIso(c, opts) } -func isoOfflineInstallFromRam(c cluster.TestCluster) { +func isoOfflineInstallMpathUefi(c cluster.TestCluster) { opts := IsoTestOpts{ - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", - isOffline: true, - isISOFromRAM: true, + enableMultipath: true, + enableUefi: true, + isOffline: true, } opts.SetInsecureOnDevBuild() isoLiveIso(c, opts) @@ -638,7 +442,6 @@ func isoOfflineInstallFromRam(c cluster.TestCluster) { func isoOfflineInstallFromRam4k(c cluster.TestCluster) { opts := IsoTestOpts{ enable4k: true, - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", isOffline: true, isISOFromRAM: true, } @@ -665,11 +468,19 @@ func isoMinisoInstall(c cluster.TestCluster) { isoLiveIso(c, opts) } -func isoMinisoInstall4k(c cluster.TestCluster) { +func isoMinisoInstallUefi(c cluster.TestCluster) { opts := IsoTestOpts{ - enable4k: true, isMiniso: true, - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + +func isoMinisoInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + isMiniso: true, } opts.SetInsecureOnDevBuild() isoLiveIso(c, opts) @@ -694,6 +505,16 @@ func isoMinisoInstallNm(c cluster.TestCluster) { isoLiveIso(c, opts) } +func isoMinisoInstallNmUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + addNmKeyfile: true, + isMiniso: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoLiveIso(c, opts) +} + func isoMinisoInstall4kNm(c cluster.TestCluster) { opts := IsoTestOpts{ addNmKeyfile: true, @@ -716,8 +537,14 @@ func isoMinisoInstall4kNmUefi(c cluster.TestCluster) { } func isoPxeOnlineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstallUefi(c cluster.TestCluster) { opts := IsoTestOpts{ - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + enableUefi: true, } opts.SetInsecureOnDevBuild() testPXE(c, opts) @@ -735,7 +562,6 @@ func isoPxeOnlineInstall4kUefi(c cluster.TestCluster) { func isoPxeOnlineInstallRootfsAppended(c cluster.TestCluster) { opts := IsoTestOpts{ pxeAppendRootfs: true, - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", } opts.SetInsecureOnDevBuild() testPXE(c, opts) @@ -781,18 +607,17 @@ func isoPxeOfflineInstallRootfsAppended(c cluster.TestCluster) { opts := IsoTestOpts{ isOffline: true, pxeAppendRootfs: true, - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", } opts.SetInsecureOnDevBuild() testPXE(c, opts) } -func isoPxeOfflineInstallRootfsAppended4k(c cluster.TestCluster) { +func isoPxeOfflineInstallRootfsAppended4kUefi(c cluster.TestCluster) { opts := IsoTestOpts{ isOffline: true, pxeAppendRootfs: true, enable4k: true, - enableUefi: coreosarch.CurrentRpmArch() == "aarch64", + enableUefi: true, } opts.SetInsecureOnDevBuild() testPXE(c, opts) diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index 72ea6f071d..b83b6698e9 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -19,33 +19,9 @@ import ( ) func init() { - register.RegisterTest(®ister.Test{ - Run: isoLiveLogin, - ClusterSize: 0, - Name: "iso.live-login", - Description: "Verify ISO live login works.", - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - ExcludeArchitectures: []string{}, - }) - register.RegisterTest(®ister.Test{ - Run: isoLiveLoginUefi, - ClusterSize: 0, - Name: "iso.live-login.uefi", - Description: "Verify ISO live login works.", - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - ExcludeArchitectures: []string{"s390x", "ppcfw"}, - }) - register.RegisterTest(®ister.Test{ - Run: isoLiveLoginUefiSecure, - ClusterSize: 0, - Name: "iso.live-login.uefi-secure", - Description: "Verify ISO live login works.", - Flags: []register.Flag{}, - Platforms: []string{"qemu"}, - ExcludeArchitectures: []string{"s390x", "ppcfw"}, - }) + register.RegisterTest(isoTest("live-login", isoLiveLogin, []string{})) + register.RegisterTest(isoTest("live-login.uefi", isoLiveLoginUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("live-login.uefi-secure", isoLiveLoginUefiSecure, []string{"x86_64", "aarch64"})) } func testLiveLogin(c cluster.TestCluster, enableUefi bool, enableUefiSecure bool) { From d525e452ae0a3b2b4beaeece41b9a29a4453a840 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 10:48:48 +0100 Subject: [PATCH 09/27] kola: fold iso *iscsi* tests --- mantle/cmd/kola/testiso.go | 167 +---------------- .../tests/iso}/iscsi_butane_setup.yaml | 0 mantle/kola/tests/iso/live-iso.go | 174 ++++++++++++++++++ 3 files changed, 176 insertions(+), 165 deletions(-) rename mantle/{cmd/kola/resources => kola/tests/iso}/iscsi_butane_setup.yaml (100%) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index ee8219341f..abd0452c85 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -63,10 +63,8 @@ var ( console bool enable4k bool - enableMultipath bool enableUefi bool enableUefiSecure bool - isOffline bool // These tests only run on RHCOS tests_RHCOS_uefi = []string{ @@ -80,27 +78,6 @@ var ( "iso-as-disk.uefi", "iso-as-disk.uefi-secure", "iso-as-disk.4k.uefi", - "iso-offline-install-iscsi.ibft.uefi", - "iso-offline-install-iscsi.ibft-with-mpath.bios", - "iso-offline-install-iscsi.manual.bios", - } - tests_s390x = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.s390fw, - //"iso-offline-install-iscsi.ibft-with-mpath.s390fw", - //"iso-offline-install-iscsi.manual.s390fw", - } - tests_ppc64le = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.ppcfw", - //"iso-offline-install-iscsi.ibft-with-mpath.ppcfw", - //"iso-offline-install-iscsi.manual.ppcfw", - } - tests_aarch64 = []string{ - // FIXME https://github.com/coreos/fedora-coreos-tracker/issues/1657 - //"iso-offline-install-iscsi.ibft.uefi", - //"iso-offline-install-iscsi.ibft-with-mpath.uefi", - //"iso-offline-install-iscsi.manual.uefi", } ) @@ -157,9 +134,6 @@ RequiredBy=coreos-installer.target # for iso-as-disk RequiredBy=multi-user.target` -//go:embed resources/iscsi_butane_setup.yaml -var iscsi_butane_config string - func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") @@ -182,12 +156,8 @@ func getAllTests(build *util.LocalBuild) []string { switch arch { case "x86_64": tests = tests_x86_64 - case "ppc64le": - tests = tests_ppc64le - case "s390x": - tests = tests_s390x - case "aarch64": - tests = tests_aarch64 + default: + return []string{} } if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { tests = append(tests, tests_RHCOS_uefi...) @@ -359,10 +329,8 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } enable4k = false - enableMultipath = false enableUefi = false enableUefiSecure = false - isOffline = false inst := baseInst // Pretend this is Rust and I wrote .copy() fmt.Printf("Running test: %s\n", test) @@ -374,40 +342,17 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { enable4k = true inst.Native4k = true } - if kola.HasString("mpath", components) { - enableMultipath = true - inst.MultiPathDisk = true - } if kola.HasString("uefi-secure", components) { enableUefiSecure = true } else if kola.HasString("uefi", components) { enableUefi = true } - // For offline it is a part of the first component. i.e. for - // iso-offline-install.bios we need to search for 'offline' in - // iso-offline-install, which is currently in components[0]. - if kola.HasString("offline", strings.Split(components[0], "-")) { - isOffline = true - } switch components[0] { case "iso-as-disk": duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - case "iso-offline-install-iscsi": - var butane_config string - switch components[1] { - case "ibft": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") - case "manual": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") - case "ibft-with-mpath": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") - default: - plog.Fatalf("Unknown test name:%s", test) - } - duration, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test), butane_config) default: plog.Fatalf("Unknown test name:%s", test) } @@ -654,111 +599,3 @@ func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } - -// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup -// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" -// - Format and mount `virtio-var` to /var -// -// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli -// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: -// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target -// - disables authentication -// - verifies the iscsi service is active and reachable -// -// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: -// - mount iscsi target -// - run coreos-installer on the mounted block device -// - unmount iscsi -// -// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: -// - launch kola qemuexec instructing it to boot from an iPXE script -// wich in turns mount the iscsi target and load kernel -// - note the virtserial port device: we pass through the serial port -// that was created by kola for test completion -// -// 6 - /var/nested-ign.json contains an ignition config: -// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion -// - as this serial device is mapped to the host serial device, the test concludes -func testLiveInstalliscsi(ctx context.Context, inst qemu.Install, outdir string, butane string) (time.Duration, error) { - - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - // Create a serial channel to read the logs from the nested VM - nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") - if err != nil { - return 0, err - } - - // Create a file to write the contents of the serial channel into - nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return 0, err - } - - go func() { - _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) - if err != nil && err != io.EOF { - panic(err) - } - }() - - // empty disk to use as an iscsi target to install coreOS on and subseqently boot - // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers - err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) - if err != nil { - return 0, err - } - - // We need more memory to start another VM within ! - builder.MemoryMiB = 2048 - - var iscsiTargetConfig = conf.Butane(butane) - - config, err := iscsiTargetConfig.Render(conf.FailWarnings) - if err != nil { - return 0, err - } - err = forwardJournal(outdir, builder, config) - if err != nil { - return 0, err - } - - // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout - config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - // enable network - builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") - - // keep auto-login enabled for easier debug when running console - config.AddAutoLogin() - - builder.SetConfig(config) - - // Bind mount in the COSA rootfs into the VM so we can use it as a - // read-only rootfs for quickly starting the container to kola - // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml - builder.MountHost("/", "/var/cosaroot", true) - config.MountHost("/var/cosaroot", true) - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) -} diff --git a/mantle/cmd/kola/resources/iscsi_butane_setup.yaml b/mantle/kola/tests/iso/iscsi_butane_setup.yaml similarity index 100% rename from mantle/cmd/kola/resources/iscsi_butane_setup.yaml rename to mantle/kola/tests/iso/iscsi_butane_setup.yaml diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index ab92cdc15c..6d7c6e6397 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -2,6 +2,7 @@ package testiso import ( "bufio" + _ "embed" "fmt" "io" "os" @@ -52,6 +53,11 @@ func init() { register.RegisterTest(isoTest("offline-install-fromram.4k", isoOfflineInstallFromRam4k, []string{"ppc64le"})) register.RegisterTest(isoTest("offline-install-fromram.4k.uefi", isoOfflineInstallFromRam4kUefi, []string{"x86_64", "aarch64"})) + // Those currently work only on x86, see: https://github.com/coreos/fedora-coreos-tracker/issues/1657 + register.RegisterTest(isoTest("offline-install-iscsi.ibft.uefi", isoOfflineInstallIscsiIbftUefi, []string{"x86_64"})) + register.RegisterTest(isoTest("offline-install-iscsi.ibft-with-mpath", isoOfflineInstallIscsiIbftMpath, []string{"x86_64"})) + register.RegisterTest(isoTest("offline-install-iscsi.manual", isoOfflineInstallIscsiManual, []string{"x86_64"})) + register.RegisterTest(isoTest("miniso-install", isoMinisoInstall, []string{"x86_64", "s390x", "ppc64le"})) register.RegisterTest(isoTest("miniso-install.uefi", isoMinisoInstallUefi, []string{"aarch64"})) register.RegisterTest(isoTest("miniso-install.4k", isoMinisoInstall4k, []string{"ppc64le"})) @@ -269,6 +275,8 @@ type IsoTestOpts struct { isMiniso bool enableUefi bool enableUefiSecure bool + enableIbft bool + manual bool pxeAppendRootfs bool } @@ -460,6 +468,37 @@ func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { isoLiveIso(c, opts) } +func isoOfflineInstallIscsiIbftUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiIbftMpath(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableMultipath: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiManual(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + manual: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + func isoMinisoInstall(c cluster.TestCluster) { opts := IsoTestOpts{ isMiniso: true, @@ -813,6 +852,141 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { } } +//go:embed iscsi_butane_setup.yaml +var iscsi_butane_config string + +// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup +// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" +// - Format and mount `virtio-var` to /var +// +// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli +// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: +// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target +// - disables authentication +// - verifies the iscsi service is active and reachable +// +// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: +// - mount iscsi target +// - run coreos-installer on the mounted block device +// - unmount iscsi +// +// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: +// - launch kola qemuexec instructing it to boot from an iPXE script +// wich in turns mount the iscsi target and load kernel +// - note the virtserial port device: we pass through the serial port +// that was created by kola for test completion +// +// 6 - /var/nested-ign.json contains an ignition config: +// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion +// - as this serial device is mapped to the host serial device, the test concludes +func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + var butane string + if opts.enableIbft && opts.enableMultipath { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") + } else if opts.enableIbft { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") + } else if opts.manual { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") + } + + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + } + builddir := kola.CosaBuild.Dir + isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + // Create a serial channel to read the logs from the nested VM + nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + if err != nil { + c.Fatal(err) + } + + // Create a file to write the contents of the serial channel into + nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + c.Fatal(err) + } + + go func() { + _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) + if err != nil && err != io.EOF { + panic(err) + } + }() + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) + if err != nil { + c.Fatal(err) + } + + // We need more memory to start another VM within ! + builder.MemoryMiB = 2048 + + var iscsiTargetConfig = conf.Butane(butane) + + config, err := iscsiTargetConfig.Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + err = forwardJournal(outdir, builder, config) + if err != nil { + c.Fatal(err) + } + + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + // enable network + builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") + + // keep auto-login enabled for easier debug when running console + config.AddAutoLogin() + + builder.SetConfig(config) + + // Bind mount in the COSA rootfs into the VM so we can use it as a + // read-only rootfs for quickly starting the container to kola + // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml + builder.MountHost("/", "/var/cosaroot", true) + config.MountHost("/var/cosaroot", true) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) + if err != nil { + c.Fatal(err) + } +} + func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { ctx := c.Context() From 4d2f56765f4c8d30e72a929b205170e352875596 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 11:05:56 +0100 Subject: [PATCH 10/27] kola: fold iso.as-disk* tests --- mantle/cmd/kola/testiso.go | 123 +++--------------------------- mantle/kola/tests/iso/live-iso.go | 80 +++++++++++++++++++ 2 files changed, 89 insertions(+), 114 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index abd0452c85..6f78e2da59 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -22,7 +22,6 @@ package main import ( "bufio" "context" - _ "embed" "fmt" "io" "os" @@ -36,7 +35,6 @@ import ( "github.com/coreos/coreos-assembler/mantle/harness/testresult" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" @@ -56,29 +54,14 @@ var ( SilenceUsage: true, } - instInsecure bool - + instInsecure bool pxeKernelArgs []string - - console bool - - enable4k bool - enableUefi bool - enableUefiSecure bool - + console bool + enableUefi bool // These tests only run on RHCOS tests_RHCOS_uefi = []string{ "iso-fips.uefi", } - - // The iso-as-disk tests are only supported in x86_64 because other - // architectures don't have the required hybrid partition table. - tests_x86_64 = []string{ - "iso-as-disk.bios", - "iso-as-disk.uefi", - "iso-as-disk.uefi-secure", - "iso-as-disk.4k.uefi", - } ) const ( @@ -116,24 +99,6 @@ ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && s RequiredBy=emergency.target `, signalEmergencyString) -// This test is broken. Please fix! -// https://github.com/coreos/coreos-assembler/issues/3554 -var verifyNoEFIBootEntry = `[Unit] -Description=TestISO Verify No EFI Boot Entry -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionPathExists=/sys/firmware/efi -Before=live-signal-ok.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target` - func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") @@ -150,26 +115,17 @@ func liveArtifactExistsInBuild() error { return nil } -func getAllTests(build *util.LocalBuild) []string { +func getAllTests() []string { arch := coreosarch.CurrentRpmArch() - var tests []string - switch arch { - case "x86_64": - tests = tests_x86_64 - default: - return []string{} - } if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { - tests = append(tests, tests_RHCOS_uefi...) + return tests_RHCOS_uefi } - return tests + return []string{} } func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { builder := qemu.NewMetalQemuBuilderDefault() - if enableUefiSecure { - builder.Firmware = "uefi-secure" - } else if enableUefi { + if enableUefi { builder.Firmware = "uefi" } @@ -249,7 +205,7 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { if kola.CosaBuild == nil { return fmt.Errorf("Must provide --build") } - tests := getAllTests(kola.CosaBuild) + tests := getAllTests() if len(args) != 0 { if tests, err = filterTests(tests, args); err != nil { return err @@ -300,23 +256,6 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { } }() - baseInst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - } - - if instInsecure { - baseInst.Insecure = true - fmt.Printf("Ignoring verification of signature on metal image\n") - } - - // Ignore signing verification by default when running with development build - // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if !baseInst.Insecure && strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { - baseInst.Insecure = true - fmt.Printf("Detected development build; disabling signature verification\n") - } - var duration time.Duration atLeastOneFailed := false @@ -328,29 +267,16 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { return err } - enable4k = false enableUefi = false - enableUefiSecure = false - inst := baseInst // Pretend this is Rust and I wrote .copy() fmt.Printf("Running test: %s\n", test) components := strings.Split(test, ".") - inst.PxeAppendRootfs = kola.HasString("rootfs-appended", components) - - if kola.HasString("4k", components) { - enable4k = true - inst.Native4k = true - } - if kola.HasString("uefi-secure", components) { - enableUefiSecure = true - } else if kola.HasString("uefi", components) { + if kola.HasString("uefi", components) { enableUefi = true } switch components[0] { - case "iso-as-disk": - duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) case "iso-fips": duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) default: @@ -568,34 +494,3 @@ RequiredBy=fips-signal-ok.service return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) } - -func testAsDisk(ctx context.Context, outdir string) (time.Duration, error) { - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", true); err != nil { - return 0, err - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - builder.SetConfig(config) - - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) -} diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 6d7c6e6397..8bd650e51a 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -42,6 +42,13 @@ func isoTest(name string, run func(c cluster.TestCluster), arch []string) *regis } func init() { + // The iso-as-disk tests are only supported in x86_64 because other + // architectures don't have the required hybrid partition table. + register.RegisterTest(isoTest("as-disk", isoAsDisk, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.uefi", isoAsDiskUefi, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.uefi-secure", isoAsDiskUefiSecure, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.4k.uefi", isoAsDisk4kUefi, []string{"x86_64"})) + register.RegisterTest(isoTest("install", isoInstall, []string{"x86_64"})) register.RegisterTest(isoTest("offline-install", isoOfflineInstall, []string{"x86_64", "s390x", "ppc64le"})) @@ -396,6 +403,36 @@ func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuil return builder, config, nil } +func isoAsDisk(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + +func isoAsDiskUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} +func isoAsDiskUefiSecure(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefiSecure: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + +func isoAsDisk4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + func isoInstall(c cluster.TestCluster) { opts := IsoTestOpts{} opts.SetInsecureOnDevBuild() @@ -1102,3 +1139,46 @@ func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console } return err } + +func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, config, err := newQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default + if err := builder.AddIso(isopath, "", true); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + builder.SetConfig(config) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + if err != nil { + c.Fatal(err) + } +} From c4d5939c4ac014f508ed2771c58763115d10c2f4 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 11:29:06 +0100 Subject: [PATCH 11/27] kola: split live-iso.go into several files --- mantle/kola/tests/iso/common.go | 469 +++++++++++++ mantle/kola/tests/iso/live-as-disk.go | 93 +++ mantle/kola/tests/iso/live-iscsi.go | 189 +++++ mantle/kola/tests/iso/live-iso.go | 977 +------------------------- mantle/kola/tests/iso/live-login.go | 2 +- mantle/kola/tests/iso/live-pxe.go | 214 ++++++ 6 files changed, 1000 insertions(+), 944 deletions(-) create mode 100644 mantle/kola/tests/iso/common.go create mode 100644 mantle/kola/tests/iso/live-as-disk.go create mode 100644 mantle/kola/tests/iso/live-iscsi.go create mode 100644 mantle/kola/tests/iso/live-pxe.go diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go new file mode 100644 index 0000000000..516f3eaaec --- /dev/null +++ b/mantle/kola/tests/iso/common.go @@ -0,0 +1,469 @@ +package iso + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" +) + +const ( + installTimeoutMins = 12 + // https://github.com/coreos/fedora-coreos-config/pull/2544 + liveISOFromRAMKarg = "coreos.liveiso.fromram" +) + +type IsoTestOpts struct { + // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") + instInsecure bool + // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") + pxeKernelArgs []string + // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") + console bool + addNmKeyfile bool + enable4k bool + enableMultipath bool + isOffline bool + isISOFromRAM bool + isMiniso bool + enableUefi bool + enableUefiSecure bool + enableIbft bool + manual bool + pxeAppendRootfs bool +} + +func (o *IsoTestOpts) SetInsecureOnDevBuild() { + // Ignore signing verification by default when running with development build + // https://github.com/coreos/fedora-coreos-tracker/issues/908 + if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + o.instInsecure = true + //fmt.Printf("Detected development build; disabling signature verification\n") + } +} + +func isoTest(name string, run func(c cluster.TestCluster), arch []string) *register.Test { + return ®ister.Test{ + Run: run, + ClusterSize: 0, + Name: "iso." + name, + Timeout: installTimeoutMins * time.Minute, + Platforms: []string{"qemu"}, + Architectures: arch, + } +} + +func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { + builder := qemu.NewMetalQemuBuilderDefault() + if opts.enableUefiSecure { + builder.Firmware = "uefi-secure" + } else if opts.enableUefi { + builder.Firmware = "uefi" + } + + if err := os.MkdirAll(outdir, 0755); err != nil { + return nil, err + } + + builder.InheritConsole = opts.console + if !opts.console { + builder.ConsoleFile = filepath.Join(outdir, "console.txt") + } + + if kola.QEMUOptions.Memory != "" { + parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) + if err != nil { + return nil, err + } + builder.MemoryMiB = int(parsedMem) + } + + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + + return builder, nil +} + +func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + return nil, nil, err + } + + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return nil, nil, err + } + + err = forwardJournal(outdir, builder, config) + if err != nil { + return nil, nil, err + } + + return builder, config, nil +} + +func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { + journalPipe, err := builder.VirtioJournal(config, "") + if err != nil { + return err + } + journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + + go func() { + _, err := io.Copy(journalOut, journalPipe) + if err != nil && err != io.EOF { + panic(err) + } + }() + + return nil +} + +func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { + builder, config, err := newQemuBuilder(opts, outdir) + + if err != nil { + return nil, nil, err + } + + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return nil, nil, err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return nil, nil, err + } + } + + return builder, config, nil +} + +func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { + ctx := c.Context() + + errchan := make(chan error) + go func() { + timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 + time.Sleep(timeout) + errchan <- fmt.Errorf("timed out after %v", timeout) + }() + if !console { + go func() { + errBuf, err := inst.WaitIgnitionError(ctx) + if err == nil { + if errBuf != "" { + c.Logf("entered emergency.target in initramfs") + path := filepath.Join(outdir, "ignition-virtio-dump.txt") + if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { + c.Errorf("Failed to write journal: %v", err) + } + err = platform.ErrInitramfsEmergency + } + } + if err != nil { + errchan <- err + } + }() + } + go func() { + err := inst.Wait() + // only one Wait() gets process data, so also manually check for signal + //plog.Debugf("qemu exited err=%v", err) + if err == nil && inst.Signaled() { + err = errors.New("process killed") + } + if err != nil { + errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") + } + time.Sleep(1 * time.Minute) + errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") + }() + go func() { + r := bufio.NewReader(qchan) + for _, exp := range expected { + l, err := r.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line := strings.TrimSpace(l) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) + return + } + } + // OK! + errchan <- nil + }() + go func() { + //check for error when switching boot order + if booterrchan != nil { + if err := <-booterrchan; err != nil { + errchan <- err + } + } + }() + err := <-errchan + if err == nil { + // No error so far, check the console and journal files + consoleFile := filepath.Join(outdir, "console.txt") + journalFile := filepath.Join(outdir, "journal.txt") + files := []string{consoleFile, journalFile} + for _, file := range files { + fileName := filepath.Base(file) + // Check if the file exists + _, err := os.Stat(file) + if os.IsNotExist(err) { + fmt.Printf("The file: %v does not exist\n", fileName) + continue + } else if err != nil { + fmt.Println(err) + return err + } + // Read the contents of the file + fileContent, err := os.ReadFile(file) + if err != nil { + fmt.Println(err) + return err + } + // Check for badness with CheckConsole + warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) + if len(badlines) > 0 { + for _, badline := range badlines { + if warnOnly { + c.Errorf("bad log line detected: %v", badline) + } else { + c.Logf("bad log line detected: %v", badline) + } + } + if !warnOnly { + err = fmt.Errorf("errors found in log files") + return err + } + } + } + } + return err +} + +var liveOKSignal = "live-test-OK" +var liveSignalOKUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Live ISO Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target`, liveOKSignal) + +var signalCompleteString = "coreos-installer-test-OK" +var signalCompletionUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Completion +Requires=dev-virtio\\x2dports-testisocompletion.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=multi-user.target`, signalCompleteString) + +var signalEmergencyString = "coreos-installer-test-entered-emergency-target" +var signalFailureUnit = fmt.Sprintf(` +[Unit] +Description=TestISO Signal Failure +Requires=dev-virtio\\x2dports-testisocompletion.device +DefaultDependencies=false +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' +[Install] +RequiredBy=emergency.target`, signalEmergencyString) + +var multipathedRoot = `[Unit] +Description=TestISO Verify Multipathed Root +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' +[Install] +RequiredBy=multi-user.target` + +var checkNoIgnition = ` +[Unit] +Description=TestISO Verify No Ignition Config +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=coreos-test-installer.service +After=coreos-ignition-firstboot-complete.service +RequiresMountsFor=/boot +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' +[Install] +RequiredBy=multi-user.target` + +// This test is broken. Please fix! +// https://github.com/coreos/coreos-assembler/issues/3554 +var verifyNoEFIBootEntry = ` +[Unit] +Description=TestISO Verify No EFI Boot Entry +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionPathExists=/sys/firmware/efi +Before=live-signal-ok.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' +[Install] +# for install tests +RequiredBy=coreos-installer.target +# for iso-as-disk +RequiredBy=multi-user.target` + +// Verify that the volume ID is the OS name. See also +// https://github.com/openshift/assisted-image-service/pull/477. +// This is the same as the LABEL of the block device for ISO9660. See +// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 +var verifyIsoVolumeId = ` +[Unit] +Description=Verify ISO Volume ID +OnFailure=emergency.target +OnFailureJobMode=isolate +# only if we're actually mounting the ISO +ConditionPathIsMountPoint=/run/media/iso +[Service] +Type=oneshot +RemainAfterExit=yes +# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso +ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" +[Install] +RequiredBy=coreos-installer.target` + +// Unit to check that /run/media/iso is not mounted when +// coreos.liveiso.fromram kernel argument is passed +var isoNotMountedUnit = ` +[Unit] +Description=Verify ISO is not mounted when coreos.liveiso.fromram +OnFailure=emergency.target +OnFailureJobMode=isolate +ConditionKernelCommandLine=coreos.liveiso.fromram +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +RemainAfterExit=yes +# Would like to use SuccessExitStatus but it doesn't support what +# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 +ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" +[Install] +RequiredBy=coreos-installer.target` + +var nmConnectionId = "CoreOS DHCP" +var nmConnectionFile = "coreos-dhcp.nmconnection" +var nmConnection = fmt.Sprintf(`[connection] +id=%s +type=ethernet +# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will +# wait for a device to be present before exiting. See +# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 +wait-device-timeout=20000 + +[ipv4] +method=auto +`, nmConnectionId) + +var nmstateConfigFile = "/etc/nmstate/br-ex.yml" +var nmstateConfig = `interfaces: + - name: br-ex + type: linux-bridge + state: up + ipv4: + enabled: false + ipv6: + enabled: false + bridge: + port: [] +` + +// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. +var verifyNmKeyfile = fmt.Sprintf(`[Unit] +Description=TestISO Verify NM Keyfile Propagation +OnFailure=emergency.target +OnFailureJobMode=isolate +Wants=network-online.target +After=network-online.target +Before=live-signal-ok.service +Before=coreos-test-installer.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" +ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s +# Also verify nmstate config +ExecStart=/usr/bin/nmcli c show br-ex +[Install] +# for live system +RequiredBy=coreos-installer.target +# for target system +RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go new file mode 100644 index 0000000000..bd1ac2015a --- /dev/null +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -0,0 +1,93 @@ +package iso + +import ( + "path/filepath" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +func init() { + // The iso-as-disk tests are only supported in x86_64 because other + // architectures don't have the required hybrid partition table. + register.RegisterTest(isoTest("as-disk", isoAsDisk, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.uefi", isoAsDiskUefi, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.uefi-secure", isoAsDiskUefiSecure, []string{"x86_64"})) + register.RegisterTest(isoTest("as-disk.4k.uefi", isoAsDisk4kUefi, []string{"x86_64"})) +} + +func isoAsDisk(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + +func isoAsDiskUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} +func isoAsDiskUefiSecure(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefiSecure: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + +func isoAsDisk4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + isoTestAsDisk(c, opts) +} + +func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, config, err := newQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default + if err := builder.AddIso(isopath, "", true); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + builder.SetConfig(config) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + if err != nil { + c.Fatal(err) + } +} diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go new file mode 100644 index 0000000000..80229b9f68 --- /dev/null +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -0,0 +1,189 @@ +package iso + +import ( + _ "embed" + "io" + "os" + "path/filepath" + "strings" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +func init() { + // Those currently work only on x86, see: https://github.com/coreos/fedora-coreos-tracker/issues/1657 + register.RegisterTest(isoTest("offline-install-iscsi.ibft.uefi", isoOfflineInstallIscsiIbftUefi, []string{"x86_64"})) + register.RegisterTest(isoTest("offline-install-iscsi.ibft-with-mpath", isoOfflineInstallIscsiIbftMpath, []string{"x86_64"})) + register.RegisterTest(isoTest("offline-install-iscsi.manual", isoOfflineInstallIscsiManual, []string{"x86_64"})) +} + +func isoOfflineInstallIscsiIbftUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiIbftMpath(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + isOffline: true, + enableMultipath: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +func isoOfflineInstallIscsiManual(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + manual: true, + enableIbft: true, + } + opts.SetInsecureOnDevBuild() + isoInstalliScsi(c, opts) +} + +//go:embed iscsi_butane_setup.yaml +var iscsi_butane_config string + +// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup +// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" +// - Format and mount `virtio-var` to /var +// +// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli +// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: +// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target +// - disables authentication +// - verifies the iscsi service is active and reachable +// +// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: +// - mount iscsi target +// - run coreos-installer on the mounted block device +// - unmount iscsi +// +// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: +// - launch kola qemuexec instructing it to boot from an iPXE script +// wich in turns mount the iscsi target and load kernel +// - note the virtserial port device: we pass through the serial port +// that was created by kola for test completion +// +// 6 - /var/nested-ign.json contains an ignition config: +// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion +// - as this serial device is mapped to the host serial device, the test concludes +func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + var butane string + if opts.enableIbft && opts.enableMultipath { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") + } else if opts.enableIbft { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") + } else if opts.manual { + butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") + } + + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + } + builddir := kola.CosaBuild.Dir + isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, err := newBaseQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + c.Fatal(err) + } + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + // Create a serial channel to read the logs from the nested VM + nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + if err != nil { + c.Fatal(err) + } + + // Create a file to write the contents of the serial channel into + nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + c.Fatal(err) + } + + go func() { + _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) + if err != nil && err != io.EOF { + panic(err) + } + }() + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) + if err != nil { + c.Fatal(err) + } + + // We need more memory to start another VM within ! + builder.MemoryMiB = 2048 + + var iscsiTargetConfig = conf.Butane(butane) + + config, err := iscsiTargetConfig.Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + err = forwardJournal(outdir, builder, config) + if err != nil { + c.Fatal(err) + } + + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout + config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + // enable network + builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") + + // keep auto-login enabled for easier debug when running console + config.AddAutoLogin() + + builder.SetConfig(config) + + // Bind mount in the COSA rootfs into the VM so we can use it as a + // read-only rootfs for quickly starting the container to kola + // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml + builder.MountHost("/", "/var/cosaroot", true) + config.MountHost("/var/cosaroot", true) + + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) + if err != nil { + c.Fatal(err) + } +} diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 8bd650e51a..c374262979 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -1,54 +1,20 @@ -package testiso +package iso import ( - "bufio" _ "embed" "fmt" - "io" "os" - "path/filepath" - "strconv" "strings" - "time" - - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" "github.com/coreos/coreos-assembler/mantle/kola" - "github.com/coreos/coreos-assembler/mantle/platform" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" - "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/coreos/coreos-assembler/mantle/util" ) -const ( - installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" -) - -func isoTest(name string, run func(c cluster.TestCluster), arch []string) *register.Test { - return ®ister.Test{ - Run: run, - ClusterSize: 0, - Name: "iso." + name, - Timeout: installTimeoutMins * time.Minute, - Platforms: []string{"qemu"}, - Architectures: arch, - } -} - func init() { - // The iso-as-disk tests are only supported in x86_64 because other - // architectures don't have the required hybrid partition table. - register.RegisterTest(isoTest("as-disk", isoAsDisk, []string{"x86_64"})) - register.RegisterTest(isoTest("as-disk.uefi", isoAsDiskUefi, []string{"x86_64"})) - register.RegisterTest(isoTest("as-disk.uefi-secure", isoAsDiskUefiSecure, []string{"x86_64"})) - register.RegisterTest(isoTest("as-disk.4k.uefi", isoAsDisk4kUefi, []string{"x86_64"})) - register.RegisterTest(isoTest("install", isoInstall, []string{"x86_64"})) register.RegisterTest(isoTest("offline-install", isoOfflineInstall, []string{"x86_64", "s390x", "ppc64le"})) @@ -59,12 +25,6 @@ func init() { register.RegisterTest(isoTest("offline-install-fromram.4k", isoOfflineInstallFromRam4k, []string{"ppc64le"})) register.RegisterTest(isoTest("offline-install-fromram.4k.uefi", isoOfflineInstallFromRam4kUefi, []string{"x86_64", "aarch64"})) - - // Those currently work only on x86, see: https://github.com/coreos/fedora-coreos-tracker/issues/1657 - register.RegisterTest(isoTest("offline-install-iscsi.ibft.uefi", isoOfflineInstallIscsiIbftUefi, []string{"x86_64"})) - register.RegisterTest(isoTest("offline-install-iscsi.ibft-with-mpath", isoOfflineInstallIscsiIbftMpath, []string{"x86_64"})) - register.RegisterTest(isoTest("offline-install-iscsi.manual", isoOfflineInstallIscsiManual, []string{"x86_64"})) - register.RegisterTest(isoTest("miniso-install", isoMinisoInstall, []string{"x86_64", "s390x", "ppc64le"})) register.RegisterTest(isoTest("miniso-install.uefi", isoMinisoInstallUefi, []string{"aarch64"})) register.RegisterTest(isoTest("miniso-install.4k", isoMinisoInstall4k, []string{"ppc64le"})) @@ -74,466 +34,6 @@ func init() { register.RegisterTest(isoTest("miniso-install.nm.uefi", isoMinisoInstallNmUefi, []string{"aarch64"})) register.RegisterTest(isoTest("miniso-install.4k.nm", isoMinisoInstall4kNm, []string{"ppc64le", "s390x"})) register.RegisterTest(isoTest("miniso-install.4k.nm.uefi", isoMinisoInstall4kNmUefi, []string{"x86_64", "aarch64"})) - - register.RegisterTest(isoTest("pxe-online-install", isoPxeOnlineInstall, []string{"x86_64"})) - register.RegisterTest(isoTest("pxe-online-install.uefi", isoPxeOnlineInstallUefi, []string{"aarch64"})) - register.RegisterTest(isoTest("pxe-online-install.4k.uefi", isoPxeOnlineInstall4kUefi, []string{"x86_64", "aarch64"})) - register.RegisterTest(isoTest("pxe-online-install.rootfs-appended", isoPxeOnlineInstallRootfsAppended, []string{"ppc64le", "s390x"})) - - register.RegisterTest(isoTest("pxe-offline-install", isoPxeOfflineInstall, []string{"s390x"})) - register.RegisterTest(isoTest("pxe-offline-install.uefi", isoPxeOfflineInstallUefi, []string{"aarch64"})) - register.RegisterTest(isoTest("pxe-offline-install.4k", isoPxeOfflineInstall4k, []string{"ppc64le"})) - register.RegisterTest(isoTest("pxe-offline-install.4k.uefi", isoPxeOfflineInstall4kUefi, []string{"x86_64", "aarch64"})) - register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended", isoPxeOfflineInstallRootfsAppended, []string{"x86_64"})) - register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended.4k.uefi", isoPxeOfflineInstallRootfsAppended4kUefi, []string{"aarch64"})) -} - -var liveOKSignal = "live-test-OK" -var liveSignalOKUnit = fmt.Sprintf(` -[Unit] -Description=TestISO Signal Live ISO Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target`, liveOKSignal) - -var signalCompleteString = "coreos-installer-test-OK" -var signalCompletionUnit = fmt.Sprintf(` -[Unit] -Description=TestISO Signal Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=multi-user.target`, signalCompleteString) - -var signalEmergencyString = "coreos-installer-test-entered-emergency-target" -var signalFailureUnit = fmt.Sprintf(` -[Unit] -Description=TestISO Signal Failure -Requires=dev-virtio\\x2dports-testisocompletion.device -DefaultDependencies=false -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=emergency.target`, signalEmergencyString) - -var multipathedRoot = `[Unit] -Description=TestISO Verify Multipathed Root -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/bash -c 'lsblk -pno NAME "/dev/mapper/$(multipath -l -v 1)" | grep -qw "$(findmnt -nvr /sysroot -o SOURCE)"' -[Install] -RequiredBy=multi-user.target` - -var checkNoIgnition = ` -[Unit] -Description=TestISO Verify No Ignition Config -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-test-installer.service -After=coreos-ignition-firstboot-complete.service -RequiresMountsFor=/boot -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '[ ! -e /boot/ignition ]' -[Install] -RequiredBy=multi-user.target` - -// This test is broken. Please fix! -// https://github.com/coreos/coreos-assembler/issues/3554 -var verifyNoEFIBootEntry = ` -[Unit] -Description=TestISO Verify No EFI Boot Entry -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionPathExists=/sys/firmware/efi -Before=live-signal-ok.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target` - -// Verify that the volume ID is the OS name. See also -// https://github.com/openshift/assisted-image-service/pull/477. -// This is the same as the LABEL of the block device for ISO9660. See -// https://github.com/util-linux/util-linux/blob/643bdae8e38055e36acf2963c3416de206081507/libblkid/src/superblocks/iso9660.c#L366-L377 -var verifyIsoVolumeId = ` -[Unit] -Description=Verify ISO Volume ID -OnFailure=emergency.target -OnFailureJobMode=isolate -# only if we're actually mounting the ISO -ConditionPathIsMountPoint=/run/media/iso -[Service] -Type=oneshot -RemainAfterExit=yes -# the backing device name is arch-dependent, but we know it's mounted on /run/media/iso -ExecStart=bash -c "[[ $(findmnt -no LABEL /run/media/iso) == %s-* ]]" -[Install] -RequiredBy=coreos-installer.target` - -// Unit to check that /run/media/iso is not mounted when -// coreos.liveiso.fromram kernel argument is passed -var isoNotMountedUnit = ` -[Unit] -Description=Verify ISO is not mounted when coreos.liveiso.fromram -OnFailure=emergency.target -OnFailureJobMode=isolate -ConditionKernelCommandLine=coreos.liveiso.fromram -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -RemainAfterExit=yes -# Would like to use SuccessExitStatus but it doesn't support what -# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635 -ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi" -[Install] -RequiredBy=coreos-installer.target` - -var nmConnectionId = "CoreOS DHCP" -var nmConnectionFile = "coreos-dhcp.nmconnection" -var nmConnection = fmt.Sprintf(`[connection] -id=%s -type=ethernet -# add wait-device-timeout here so we make sure NetworkManager-wait-online.service will -# wait for a device to be present before exiting. See -# https://github.com/coreos/fedora-coreos-tracker/issues/1275#issuecomment-1231605438 -wait-device-timeout=20000 - -[ipv4] -method=auto -`, nmConnectionId) - -var nmstateConfigFile = "/etc/nmstate/br-ex.yml" -var nmstateConfig = `interfaces: - - name: br-ex - type: linux-bridge - state: up - ipv4: - enabled: false - ipv6: - enabled: false - bridge: - port: [] -` - -// This is used to verify *both* the live and the target system in the `--add-nm-keyfile` path. -var verifyNmKeyfile = fmt.Sprintf(`[Unit] -Description=TestISO Verify NM Keyfile Propagation -OnFailure=emergency.target -OnFailureJobMode=isolate -Wants=network-online.target -After=network-online.target -Before=live-signal-ok.service -Before=coreos-test-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/bin/journalctl -u nm-initrd --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/journalctl -u NetworkManager --no-pager --grep "policy: set '%[1]s' (.*) as default .* routing and DNS" -ExecStart=/usr/bin/grep "%[1]s" /etc/NetworkManager/system-connections/%[2]s -# Also verify nmstate config -ExecStart=/usr/bin/nmcli c show br-ex -[Install] -# for live system -RequiredBy=coreos-installer.target -# for target system -RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) - -type IsoTestOpts struct { - // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - instInsecure bool - // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - pxeKernelArgs []string - // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - console bool - addNmKeyfile bool - enable4k bool - enableMultipath bool - isOffline bool - isISOFromRAM bool - isMiniso bool - enableUefi bool - enableUefiSecure bool - enableIbft bool - manual bool - pxeAppendRootfs bool -} - -func (o *IsoTestOpts) SetInsecureOnDevBuild() { - // Ignore signing verification by default when running with development build - // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { - o.instInsecure = true - //fmt.Printf("Detected development build; disabling signature verification\n") - } -} - -func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { - builder := qemu.NewMetalQemuBuilderDefault() - if opts.enableUefiSecure { - builder.Firmware = "uefi-secure" - } else if opts.enableUefi { - builder.Firmware = "uefi" - } - - if err := os.MkdirAll(outdir, 0755); err != nil { - return nil, err - } - - builder.InheritConsole = opts.console - if !opts.console { - builder.ConsoleFile = filepath.Join(outdir, "console.txt") - } - - if kola.QEMUOptions.Memory != "" { - parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) - if err != nil { - return nil, err - } - builder.MemoryMiB = int(parsedMem) - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - return builder, nil -} - -func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - return nil, nil, err - } - - config, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, nil, err - } - - err = forwardJournal(outdir, builder, config) - if err != nil { - return nil, nil, err - } - - return builder, config, nil -} - -func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { - journalPipe, err := builder.VirtioJournal(config, "") - if err != nil { - return err - } - journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - go func() { - _, err := io.Copy(journalOut, journalPipe) - if err != nil && err != io.EOF { - panic(err) - } - }() - - return nil -} - -func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, config, err := newQemuBuilder(opts, outdir) - - if err != nil { - return nil, nil, err - } - - sectorSize := 0 - if opts.enable4k { - sectorSize = 4096 - } - - disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - MultiPathDisk: opts.enableMultipath, - } - - //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup - if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { - // s390x and aarch64 need to use bootindex as they don't support boot once - if err := builder.AddDisk(&disk); err != nil { - return nil, nil, err - } - } else { - if err := builder.AddPrimaryDisk(&disk); err != nil { - return nil, nil, err - } - } - - return builder, config, nil -} - -func isoAsDisk(c cluster.TestCluster) { - opts := IsoTestOpts{} - opts.SetInsecureOnDevBuild() - isoTestAsDisk(c, opts) -} - -func isoAsDiskUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableUefi: true, - } - opts.SetInsecureOnDevBuild() - isoTestAsDisk(c, opts) -} -func isoAsDiskUefiSecure(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableUefiSecure: true, - } - opts.SetInsecureOnDevBuild() - isoTestAsDisk(c, opts) -} - -func isoAsDisk4kUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enable4k: true, - enableUefi: true, - } - opts.SetInsecureOnDevBuild() - isoTestAsDisk(c, opts) -} - -func isoInstall(c cluster.TestCluster) { - opts := IsoTestOpts{} - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstall(c cluster.TestCluster) { - opts := IsoTestOpts{ - isOffline: true, - } - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstallUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableUefi: true, - isOffline: true, - } - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstall4k(c cluster.TestCluster) { - opts := IsoTestOpts{ - enable4k: true, - isOffline: true, - } - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstallMpath(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableMultipath: true, - isOffline: true, - } - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstallMpathUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableMultipath: true, - enableUefi: true, - isOffline: true, - } - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstallFromRam4k(c cluster.TestCluster) { - opts := IsoTestOpts{ - enable4k: true, - isOffline: true, - isISOFromRAM: true, - } - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enable4k: true, - enableUefi: true, - isOffline: true, - isISOFromRAM: true, - } - opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) -} - -func isoOfflineInstallIscsiIbftUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableUefi: true, - isOffline: true, - enableIbft: true, - } - opts.SetInsecureOnDevBuild() - isoInstalliScsi(c, opts) -} - -func isoOfflineInstallIscsiIbftMpath(c cluster.TestCluster) { - opts := IsoTestOpts{ - enableUefi: true, - isOffline: true, - enableMultipath: true, - enableIbft: true, - } - opts.SetInsecureOnDevBuild() - isoInstalliScsi(c, opts) -} - -func isoOfflineInstallIscsiManual(c cluster.TestCluster) { - opts := IsoTestOpts{ - isOffline: true, - manual: true, - enableIbft: true, - } - opts.SetInsecureOnDevBuild() - isoInstalliScsi(c, opts) } func isoMinisoInstall(c cluster.TestCluster) { @@ -612,91 +112,76 @@ func isoMinisoInstall4kNmUefi(c cluster.TestCluster) { isoLiveIso(c, opts) } -func isoPxeOnlineInstall(c cluster.TestCluster) { +func isoInstall(c cluster.TestCluster) { opts := IsoTestOpts{} opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } -func isoPxeOnlineInstallUefi(c cluster.TestCluster) { +func isoOfflineInstall(c cluster.TestCluster) { opts := IsoTestOpts{ - enableUefi: true, + isOffline: true, } opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } -func isoPxeOnlineInstall4kUefi(c cluster.TestCluster) { +func isoOfflineInstallUefi(c cluster.TestCluster) { opts := IsoTestOpts{ - enable4k: true, enableUefi: true, + isOffline: true, } opts.SetInsecureOnDevBuild() - testPXE(c, opts) -} - -func isoPxeOnlineInstallRootfsAppended(c cluster.TestCluster) { - opts := IsoTestOpts{ - pxeAppendRootfs: true, - } - opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } -func isoPxeOfflineInstall(c cluster.TestCluster) { +func isoOfflineInstall4k(c cluster.TestCluster) { opts := IsoTestOpts{ + enable4k: true, isOffline: true, } opts.SetInsecureOnDevBuild() - testPXE(c, opts) -} - -func isoPxeOfflineInstallUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - isOffline: true, - enableUefi: true, - } - opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } -func isoPxeOfflineInstall4k(c cluster.TestCluster) { +func isoOfflineInstallMpath(c cluster.TestCluster) { opts := IsoTestOpts{ - isOffline: true, - enable4k: true, + enableMultipath: true, + isOffline: true, } opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } -func isoPxeOfflineInstall4kUefi(c cluster.TestCluster) { +func isoOfflineInstallMpathUefi(c cluster.TestCluster) { opts := IsoTestOpts{ - isOffline: true, - enable4k: true, - enableUefi: true, + enableMultipath: true, + enableUefi: true, + isOffline: true, } opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } -func isoPxeOfflineInstallRootfsAppended(c cluster.TestCluster) { +func isoOfflineInstallFromRam4k(c cluster.TestCluster) { opts := IsoTestOpts{ - isOffline: true, - pxeAppendRootfs: true, + enable4k: true, + isOffline: true, + isISOFromRAM: true, } opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } -func isoPxeOfflineInstallRootfsAppended4kUefi(c cluster.TestCluster) { +func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { opts := IsoTestOpts{ - isOffline: true, - pxeAppendRootfs: true, - enable4k: true, - enableUefi: true, + enable4k: true, + enableUefi: true, + isOffline: true, + isISOFromRAM: true, } opts.SetInsecureOnDevBuild() - testPXE(c, opts) + isoLiveIso(c, opts) } func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { @@ -788,397 +273,3 @@ func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { c.Fatal(err) } } - -var downloadCheck = `[Unit] -Description=TestISO Verify CoreOS Installer Download -After=coreos-installer.service -Before=coreos-installer.target -[Service] -Type=oneshot -StandardOutput=kmsg+console -StandardError=kmsg+console -ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" -ExecStart=/bin/sh -c "/usr/bin/udevadm settle" -ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" -ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" -[Install] -RequiredBy=coreos-installer.target -` - -func testPXE(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - var qc *qemu.Cluster - - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - if opts.addNmKeyfile { - c.Fatal("--add-nm-keyfile not yet supported for PXE") - } - - inst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - Insecure: opts.instInsecure, - Native4k: opts.enable4k, - MultiPathDisk: opts.enableMultipath, - PxeAppendRootfs: opts.pxeAppendRootfs, - } - - tmpd, err := os.MkdirTemp("", "kola-iso.pxe") - if err != nil { - c.Fatal(err) - } - defer os.RemoveAll(tmpd) - - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { - c.Fatal(err) - } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) - if err != nil { - c.Fatal(err) - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) // , "setting up virtio-serial channel") - } - - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig - liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - if opts.isOffline { - contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) - liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) - } - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - - mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) - if err != nil { - c.Fatal(err) - } - qc.AddMach(mach) - - err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) - if err != nil { - c.Fatal(err) - } -} - -//go:embed iscsi_butane_setup.yaml -var iscsi_butane_config string - -// iscsi_butane_setup.yaml contains the full butane config but here is an overview of the setup -// 1 - Boot a live ISO with two extra 10G disks with labels "target" and "var" -// - Format and mount `virtio-var` to /var -// -// 2 - target.container -> start an iscsi target, using quay.io/coreos-assembler/targetcli -// 3 - setup-targetcli.service calls /usr/local/bin/targetcli_script: -// - instructs targetcli to serve /dev/disk/by-id/virtio-target as an iscsi target -// - disables authentication -// - verifies the iscsi service is active and reachable -// -// 4 - install-coreos-to-iscsi-target.service calls /usr/local/bin/install-coreos-iscsi: -// - mount iscsi target -// - run coreos-installer on the mounted block device -// - unmount iscsi -// -// 5 - coreos-iscsi-vm.container -> start a coreos-assembler conainer: -// - launch kola qemuexec instructing it to boot from an iPXE script -// wich in turns mount the iscsi target and load kernel -// - note the virtserial port device: we pass through the serial port -// that was created by kola for test completion -// -// 6 - /var/nested-ign.json contains an ignition config: -// - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion -// - as this serial device is mapped to the host serial device, the test concludes -func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - var butane string - if opts.enableIbft && opts.enableMultipath { - butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") - } else if opts.enableIbft { - butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") - } else if opts.manual { - butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") - } - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) - } - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - c.Fatal(err) - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - // Create a serial channel to read the logs from the nested VM - nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") - if err != nil { - c.Fatal(err) - } - - // Create a file to write the contents of the serial channel into - nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - c.Fatal(err) - } - - go func() { - _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) - if err != nil && err != io.EOF { - panic(err) - } - }() - - // empty disk to use as an iscsi target to install coreOS on and subseqently boot - // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers - err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) - if err != nil { - c.Fatal(err) - } - - // We need more memory to start another VM within ! - builder.MemoryMiB = 2048 - - var iscsiTargetConfig = conf.Butane(butane) - - config, err := iscsiTargetConfig.Render(conf.FailWarnings) - if err != nil { - c.Fatal(err) - } - err = forwardJournal(outdir, builder, config) - if err != nil { - c.Fatal(err) - } - - // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout - config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - - // enable network - builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") - - // keep auto-login enabled for easier debug when running console - config.AddAutoLogin() - - builder.SetConfig(config) - - // Bind mount in the COSA rootfs into the VM so we can use it as a - // read-only rootfs for quickly starting the container to kola - // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml - builder.MountHost("/", "/var/cosaroot", true) - config.MountHost("/var/cosaroot", true) - - mach, err := builder.Exec() - if err != nil { - c.Fatal(err) - } - defer mach.Destroy() - - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) - if err != nil { - c.Fatal(err) - } -} - -func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { - ctx := c.Context() - - errchan := make(chan error) - go func() { - timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 - time.Sleep(timeout) - errchan <- fmt.Errorf("timed out after %v", timeout) - }() - if !console { - go func() { - errBuf, err := inst.WaitIgnitionError(ctx) - if err == nil { - if errBuf != "" { - c.Logf("entered emergency.target in initramfs") - path := filepath.Join(outdir, "ignition-virtio-dump.txt") - if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { - c.Errorf("Failed to write journal: %v", err) - } - err = platform.ErrInitramfsEmergency - } - } - if err != nil { - errchan <- err - } - }() - } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - //plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") - } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return - } - } - // OK! - errchan <- nil - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err - } - } - }() - err := <-errchan - if err == nil { - // No error so far, check the console and journal files - consoleFile := filepath.Join(outdir, "console.txt") - journalFile := filepath.Join(outdir, "journal.txt") - files := []string{consoleFile, journalFile} - for _, file := range files { - fileName := filepath.Base(file) - // Check if the file exists - _, err := os.Stat(file) - if os.IsNotExist(err) { - fmt.Printf("The file: %v does not exist\n", fileName) - continue - } else if err != nil { - fmt.Println(err) - return err - } - // Read the contents of the file - fileContent, err := os.ReadFile(file) - if err != nil { - fmt.Println(err) - return err - } - // Check for badness with CheckConsole - warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) - if len(badlines) > 0 { - for _, badline := range badlines { - if warnOnly { - c.Errorf("bad log line detected: %v", badline) - } else { - c.Logf("bad log line detected: %v", badline) - } - } - if !warnOnly { - err = fmt.Errorf("errors found in log files") - return err - } - } - } - } - return err -} - -func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", true); err != nil { - c.Fatal(err) - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - builder.SetConfig(config) - - mach, err := builder.Exec() - if err != nil { - c.Fatal(err) - } - defer mach.Destroy() - - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) - if err != nil { - c.Fatal(err) - } -} diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index b83b6698e9..e2cff5d51f 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -1,4 +1,4 @@ -package testiso +package iso import ( "bufio" diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go new file mode 100644 index 0000000000..8d2acf5cd0 --- /dev/null +++ b/mantle/kola/tests/iso/live-pxe.go @@ -0,0 +1,214 @@ +package iso + +import ( + "fmt" + "os" + "strings" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/coreos/coreos-assembler/mantle/util" +) + +func init() { + register.RegisterTest(isoTest("pxe-online-install", isoPxeOnlineInstall, []string{"x86_64"})) + register.RegisterTest(isoTest("pxe-online-install.uefi", isoPxeOnlineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("pxe-online-install.4k.uefi", isoPxeOnlineInstall4kUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("pxe-online-install.rootfs-appended", isoPxeOnlineInstallRootfsAppended, []string{"ppc64le", "s390x"})) + register.RegisterTest(isoTest("pxe-offline-install", isoPxeOfflineInstall, []string{"s390x"})) + register.RegisterTest(isoTest("pxe-offline-install.uefi", isoPxeOfflineInstallUefi, []string{"aarch64"})) + register.RegisterTest(isoTest("pxe-offline-install.4k", isoPxeOfflineInstall4k, []string{"ppc64le"})) + register.RegisterTest(isoTest("pxe-offline-install.4k.uefi", isoPxeOfflineInstall4kUefi, []string{"x86_64", "aarch64"})) + register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended", isoPxeOfflineInstallRootfsAppended, []string{"x86_64"})) + register.RegisterTest(isoTest("pxe-offline-install.rootfs-appended.4k.uefi", isoPxeOfflineInstallRootfsAppended4kUefi, []string{"aarch64"})) +} + +func isoPxeOnlineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{} + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstallUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOnlineInstallRootfsAppended(c cluster.TestCluster) { + opts := IsoTestOpts{ + pxeAppendRootfs: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall4k(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enable4k: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstall4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallRootfsAppended(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + pxeAppendRootfs: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +func isoPxeOfflineInstallRootfsAppended4kUefi(c cluster.TestCluster) { + opts := IsoTestOpts{ + isOffline: true, + pxeAppendRootfs: true, + enable4k: true, + enableUefi: true, + } + opts.SetInsecureOnDevBuild() + testPXE(c, opts) +} + +var downloadCheck = `[Unit] +Description=TestISO Verify CoreOS Installer Download +After=coreos-installer.service +Before=coreos-installer.target +[Service] +Type=oneshot +StandardOutput=kmsg+console +StandardError=kmsg+console +ExecStart=/bin/sh -c "journalctl -t coreos-installer-service | /usr/bin/awk '/[Dd]ownload/ {exit 1}'" +ExecStart=/bin/sh -c "/usr/bin/udevadm settle" +ExecStart=/bin/sh -c "/usr/bin/mount /dev/disk/by-label/root /mnt" +ExecStart=/bin/sh -c "/usr/bin/jq -er '.[\"build\"]? + .[\"version\"]? == \"%s\"' /mnt/.coreos-aleph-version.json" +[Install] +RequiredBy=coreos-installer.target +` + +func testPXE(c cluster.TestCluster, opts IsoTestOpts) { + var outdir string + var qc *qemu.Cluster + + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + if opts.addNmKeyfile { + c.Fatal("--add-nm-keyfile not yet supported for PXE") + } + + inst := qemu.Install{ + CosaBuild: kola.CosaBuild, + NmKeyfiles: make(map[string]string), + Insecure: opts.instInsecure, + Native4k: opts.enable4k, + MultiPathDisk: opts.enableMultipath, + PxeAppendRootfs: opts.pxeAppendRootfs, + } + + tmpd, err := os.MkdirTemp("", "kola-iso.pxe") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpd) + + sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + if err != nil { + c.Fatal(err) + } + + builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + if err != nil { + c.Fatal(err) + } + + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + // pxe-offline-install.rootfs-appended.bios tests + if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { + builder.MemoryMiB = 5120 + } + + inst.Builder = builder + completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) // , "setting up virtio-serial channel") + } + + var keys []string + keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) + virtioJournalConfig.AddAuthorizedKeys("core", keys) + + liveConfig := *virtioJournalConfig + liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + + if opts.isOffline { + contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) + liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) + } + + targetConfig := *virtioJournalConfig + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + + mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + if err != nil { + c.Fatal(err) + } + qc.AddMach(mach) + + err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + if err != nil { + c.Fatal(err) + } +} From 12fb7e760660b32585b2ed8d68e8eeb1df69656b Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 12:28:56 +0100 Subject: [PATCH 12/27] kola: add inst-insecure and pxe-kargs arguments to QEMUOptions --- mantle/cmd/kola/options.go | 4 ++++ mantle/kola/tests/iso/common.go | 4 +--- mantle/kola/tests/iso/live-pxe.go | 2 +- mantle/platform/machine/qemu/flight.go | 5 +++++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index a63877e52c..59e145bc77 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -162,6 +162,10 @@ func init() { sv(&kola.QEMUOptions.SecureExecutionHostKey, "qemu-secex-hostkey", "", "Path to Secure Execution HKD certificate") // s390x CEX-specific options bv(&kola.QEMUOptions.Cex, "qemu-cex", false, "Attach CEX device to guest") + + // kola run iso.* options + bv(&kola.QEMUOptions.InstInsecure, "inst-insecure", false, "Do not verify signature on metal image") + ssv(&kola.QEMUOptions.PxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") } // Sync up the command line options if there is dependency diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 516f3eaaec..3459cd8260 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -29,8 +29,6 @@ const ( type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") instInsecure bool - // Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - pxeKernelArgs []string // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") console bool addNmKeyfile bool @@ -49,7 +47,7 @@ type IsoTestOpts struct { func (o *IsoTestOpts) SetInsecureOnDevBuild() { // Ignore signing verification by default when running with development build // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + if kola.QEMUOptions.InstInsecure || strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { o.instInsecure = true //fmt.Printf("Detected development build; disabling signature verification\n") } diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 8d2acf5cd0..0057becb55 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -201,7 +201,7 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - mach, err := inst.PXE(opts.pxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + mach, err := inst.PXE(kola.QEMUOptions.PxeKernelArgs, liveConfig, targetConfig, opts.isOffline) if err != nil { c.Fatal(err) } diff --git a/mantle/platform/machine/qemu/flight.go b/mantle/platform/machine/qemu/flight.go index 4a008b1a0a..bc3bd90a54 100644 --- a/mantle/platform/machine/qemu/flight.go +++ b/mantle/platform/machine/qemu/flight.go @@ -55,6 +55,11 @@ type Options struct { SecureExecutionIgnitionPubKey string SecureExecutionHostKey string + // kola run iso.* options + // Do not verify signature on metal image + InstInsecure bool + PxeKernelArgs []string + // Option to create IBM cex based luks encryption Cex bool From 842fd656d6374f8747b7ccac86d104dde29567b8 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 12:40:10 +0100 Subject: [PATCH 13/27] kola: fold iso.fips.uefi test --- mantle/cmd/kola/testiso.go | 496 ----------------------------- mantle/kola/tests/iso/live-fips.go | 92 ++++++ 2 files changed, 92 insertions(+), 496 deletions(-) delete mode 100644 mantle/cmd/kola/testiso.go create mode 100644 mantle/kola/tests/iso/live-fips.go diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go deleted file mode 100644 index 6f78e2da59..0000000000 --- a/mantle/cmd/kola/testiso.go +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2020 Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO: -// - Support testing the "just run Live" case - maybe try to figure out -// how to have main `kola` tests apply? -// - Test `coreos-install iso embed` path - -package main - -import ( - "bufio" - "context" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/coreos/coreos-assembler/mantle/harness" - "github.com/coreos/coreos-assembler/mantle/harness/reporters" - "github.com/coreos/coreos-assembler/mantle/harness/testresult" - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" - - "github.com/spf13/cobra" - - "github.com/coreos/coreos-assembler/mantle/kola" - "github.com/coreos/coreos-assembler/mantle/platform" -) - -var ( - cmdTestIso = &cobra.Command{ - RunE: runTestIso, - PreRunE: preRun, - Use: "testiso [glob pattern...]", - Short: "Test a CoreOS PXE boot or ISO install path", - - SilenceUsage: true, - } - - instInsecure bool - pxeKernelArgs []string - console bool - enableUefi bool - // These tests only run on RHCOS - tests_RHCOS_uefi = []string{ - "iso-fips.uefi", - } -) - -const ( - installTimeoutMins = 12 -) - -var liveOKSignal = "live-test-OK" -var liveSignalOKUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Live ISO Completion -Requires=dev-virtio\\x2dports-testisocompletion.device -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=coreos-installer.service -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' -[Install] -# for install tests -RequiredBy=coreos-installer.target -# for iso-as-disk -RequiredBy=multi-user.target -`, liveOKSignal) - -var signalEmergencyString = "coreos-installer-test-entered-emergency-target" -var signalFailureUnit = fmt.Sprintf(`[Unit] -Description=TestISO Signal Failure -Requires=dev-virtio\\x2dports-testisocompletion.device -DefaultDependencies=false -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion && systemctl poweroff' -[Install] -RequiredBy=emergency.target -`, signalEmergencyString) - -func init() { - cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - cmdTestIso.Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - - root.AddCommand(cmdTestIso) -} - -func liveArtifactExistsInBuild() error { - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - return fmt.Errorf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) - } - return nil -} - -func getAllTests() []string { - arch := coreosarch.CurrentRpmArch() - if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { - return tests_RHCOS_uefi - } - return []string{} -} - -func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { - builder := qemu.NewMetalQemuBuilderDefault() - if enableUefi { - builder.Firmware = "uefi" - } - - if err := os.MkdirAll(outdir, 0755); err != nil { - return nil, err - } - - builder.InheritConsole = console - if !console { - builder.ConsoleFile = filepath.Join(outdir, "console.txt") - } - - if kola.QEMUOptions.Memory != "" { - parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) - if err != nil { - return nil, err - } - builder.MemoryMiB = int(parsedMem) - } - - return builder, nil -} - -func newQemuBuilder(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, err := newBaseQemuBuilder(outdir) - if err != nil { - return nil, nil, err - } - - config, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, nil, err - } - - err = forwardJournal(outdir, builder, config) - if err != nil { - return nil, nil, err - } - - return builder, config, nil -} - -func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { - journalPipe, err := builder.VirtioJournal(config, "") - if err != nil { - return err - } - journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - go func() { - _, err := io.Copy(journalOut, journalPipe) - if err != nil && err != io.EOF { - panic(err) - } - }() - - return nil -} - -// See similar semantics in the `filterTests` of `kola.go`. -func filterTests(tests []string, patterns []string) ([]string, error) { - r := []string{} - for _, test := range tests { - if matches, err := kola.MatchesPatterns(test, patterns); err != nil { - return nil, err - } else if matches { - r = append(r, test) - } - } - return r, nil -} - -func runTestIso(cmd *cobra.Command, args []string) (err error) { - if kola.CosaBuild == nil { - return fmt.Errorf("Must provide --build") - } - tests := getAllTests() - if len(args) != 0 { - if tests, err = filterTests(tests, args); err != nil { - return err - } else if len(tests) == 0 { - return harness.SuiteEmpty - } - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Call `ParseDenyListYaml` to populate the `kola.DenylistedTests` var - err = kola.ParseDenyListYaml("qemu") - if err != nil { - plog.Fatal(err) - } - - finalTests := []string{} - for _, test := range tests { - if !kola.HasString(test, kola.DenylistedTests) { - matchTest, err := kola.MatchesPatterns(test, kola.DenylistedTests) - if err != nil { - return err - - } - if !matchTest { - finalTests = append(finalTests, test) - } - } - } - - // note this reassigns a *global* - outputDir, err = kola.SetupOutputDir(outputDir, "testiso") - if err != nil { - return err - } - - // see similar code in suite.go - reportDir := filepath.Join(outputDir, "reports") - if err := os.Mkdir(reportDir, 0777); err != nil { - return err - } - - reporter := reporters.NewJSONReporter("report.json", "testiso", "") - defer func() { - if reportErr := reporter.Output(reportDir); reportErr != nil && err != nil { - err = reportErr - } - }() - - var duration time.Duration - - atLeastOneFailed := false - for _, test := range finalTests { - - // All of these tests require buildextend-live to have been run - err = liveArtifactExistsInBuild() - if err != nil { - return err - } - - enableUefi = false - - fmt.Printf("Running test: %s\n", test) - components := strings.Split(test, ".") - - if kola.HasString("uefi", components) { - enableUefi = true - } - - switch components[0] { - case "iso-fips": - duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - default: - plog.Fatalf("Unknown test name:%s", test) - } - - result := testresult.Pass - output := []byte{} - if err != nil { - result = testresult.Fail - output = []byte(err.Error()) - } - reporter.ReportTest(test, []string{}, result, duration, output) - if printResult(test, duration, err) { - atLeastOneFailed = true - } - } - - reporter.SetResult(testresult.Pass) - if atLeastOneFailed { - reporter.SetResult(testresult.Fail) - return harness.SuiteFailed - } - - return nil -} - -func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir string, qchan *os.File, booterrchan chan error, expected []string) (time.Duration, error) { - start := time.Now() - errchan := make(chan error) - go func() { - timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 - time.Sleep(timeout) - errchan <- fmt.Errorf("timed out after %v", timeout) - }() - if !console { - go func() { - errBuf, err := inst.WaitIgnitionError(ctx) - if err == nil { - if errBuf != "" { - plog.Info("entered emergency.target in initramfs") - path := filepath.Join(outdir, "ignition-virtio-dump.txt") - if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { - plog.Errorf("Failed to write journal: %v", err) - } - err = platform.ErrInitramfsEmergency - } - } - if err != nil { - errchan <- err - } - }() - } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") - } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return - } - plog.Debugf("Matched expected message %s", exp) - } - plog.Debugf("Matched all expected messages") - // OK! - errchan <- nil - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err - } - } - }() - err := <-errchan - elapsed := time.Since(start) - if err == nil { - // No error so far, check the console and journal files - consoleFile := filepath.Join(outdir, "console.txt") - journalFile := filepath.Join(outdir, "journal.txt") - files := []string{consoleFile, journalFile} - for _, file := range files { - fileName := filepath.Base(file) - // Check if the file exists - _, err := os.Stat(file) - if os.IsNotExist(err) { - fmt.Printf("The file: %v does not exist\n", fileName) - continue - } else if err != nil { - fmt.Println(err) - return elapsed, err - } - // Read the contents of the file - fileContent, err := os.ReadFile(file) - if err != nil { - fmt.Println(err) - return elapsed, err - } - // Check for badness with CheckConsole - warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) - if len(badlines) > 0 { - for _, badline := range badlines { - if warnOnly { - plog.Errorf("bad log line detected: %v", badline) - } else { - plog.Warningf("bad log line detected: %v", badline) - } - } - if !warnOnly { - err = fmt.Errorf("errors found in log files") - return elapsed, err - } - } - } - } - return elapsed, err -} - -func printResult(test string, duration time.Duration, err error) bool { - result := "PASS" - if err != nil { - result = "FAIL" - } - fmt.Printf("%s: %s (%s)\n", result, test, duration.Round(time.Millisecond).String()) - if err != nil { - fmt.Printf(" %s\n", err) - return true - } - return false -} - -// testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system -func testLiveFIPS(ctx context.Context, outdir string) (time.Duration, error) { - tmpd, err := os.MkdirTemp("", "kola-testiso") - if err != nil { - return 0, err - } - defer os.RemoveAll(tmpd) - - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(outdir) - if err != nil { - return 0, err - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - return 0, err - } - - // This is the core change under test - adding the `fips=1` kernel argument via - // coreos-installer iso kargs modify should enter fips mode. - // Removing this line should cause this test to fail. - builder.AppendKernelArgs = "fips=1" - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - return 0, err - } - - config.AddSystemdUnit("fips-verify.service", ` -[Unit] -OnFailure=emergency.target -OnFailureJobMode=isolate -Before=fips-signal-ok.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=grep 1 /proc/sys/crypto/fips_enabled -ExecStart=grep FIPS etc/crypto-policies/config - -[Install] -RequiredBy=fips-signal-ok.service -`, conf.Enable) - config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) - - // Just for reliability, we'll run fully offline - builder.Append("-net", "none") - - builder.SetConfig(config) - mach, err := builder.Exec() - if err != nil { - return 0, errors.Wrapf(err, "running iso") - } - defer mach.Destroy() - - return awaitCompletion(ctx, mach, outdir, completionChannel, nil, []string{liveOKSignal}) -} diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go new file mode 100644 index 0000000000..39bd2396d4 --- /dev/null +++ b/mantle/kola/tests/iso/live-fips.go @@ -0,0 +1,92 @@ +package iso + +import ( + "path/filepath" + "time" + + "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform/conf" + "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" +) + +func init() { + register.RegisterTest(®ister.Test{ + Run: testLiveFIPS, + ClusterSize: 0, + Name: "iso.fips.uefi", + Timeout: installTimeoutMins * time.Minute, + Distros: []string{"rhcos"}, + Platforms: []string{"qemu"}, + Architectures: []string{"x86_64", "aarch64"}, + }) +} + +// testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system +func testLiveFIPS(c cluster.TestCluster) { + opts := IsoTestOpts{enableUefi: true} + + var outdir string + //var qc *qemu.Cluster + switch pc := c.Cluster.(type) { + case *qemu.Cluster: + outdir = pc.RuntimeConf().OutputDir + //qc = pc + default: + c.Fatalf("Unsupported cluster type") + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + builder, config, err := newQemuBuilder(opts, outdir) + if err != nil { + c.Fatal(err) + } + defer builder.Close() + if err := builder.AddIso(isopath, "", false); err != nil { + c.Fatal(err) + } + + // This is the core change under test - adding the `fips=1` kernel argument via + // coreos-installer iso kargs modify should enter fips mode. + // Removing this line should cause this test to fail. + builder.AppendKernelArgs = "fips=1" + + completionChannel, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + c.Fatal(err) + } + + config.AddSystemdUnit("fips-verify.service", ` +[Unit] +OnFailure=emergency.target +OnFailureJobMode=isolate +Before=fips-signal-ok.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=grep 1 /proc/sys/crypto/fips_enabled +ExecStart=grep FIPS etc/crypto-policies/config + +[Install] +RequiredBy=fips-signal-ok.service +`, conf.Enable) + config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) + + // Just for reliability, we'll run fully offline + builder.Append("-net", "none") + + builder.SetConfig(config) + mach, err := builder.Exec() + if err != nil { + c.Fatal(err) + } + defer mach.Destroy() + + err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + if err != nil { + c.Fatal(err) + } +} From bd3e4647149e2eb051eb57aaae39797d1cbd379c Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 21 Nov 2025 14:27:33 +0100 Subject: [PATCH 14/27] kola: refactor iso.fips.uefi test --- mantle/kola/tests/iso/live-fips.go | 119 ++++++++++++++++++----------- 1 file changed, 73 insertions(+), 46 deletions(-) diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index 39bd2396d4..70f21c65ca 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -1,14 +1,20 @@ package iso import ( + "bufio" + "fmt" + "io" "path/filepath" + "strings" "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/pkg/errors" ) func init() { @@ -16,49 +22,14 @@ func init() { Run: testLiveFIPS, ClusterSize: 0, Name: "iso.fips.uefi", - Timeout: installTimeoutMins * time.Minute, + Description: "verifies that adding fips=1 to the ISO results in a FIPS mode system", Distros: []string{"rhcos"}, Platforms: []string{"qemu"}, Architectures: []string{"x86_64", "aarch64"}, }) } -// testLiveFIPS verifies that adding fips=1 to the ISO results in a FIPS mode system -func testLiveFIPS(c cluster.TestCluster) { - opts := IsoTestOpts{enableUefi: true} - - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - - isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - c.Fatal(err) - } - - // This is the core change under test - adding the `fips=1` kernel argument via - // coreos-installer iso kargs modify should enter fips mode. - // Removing this line should cause this test to fail. - builder.AppendKernelArgs = "fips=1" - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - config.AddSystemdUnit("fips-verify.service", ` -[Unit] +var fipsVerify = `[Unit] OnFailure=emergency.target OnFailureJobMode=isolate Before=fips-signal-ok.service @@ -70,22 +41,78 @@ ExecStart=grep 1 /proc/sys/crypto/fips_enabled ExecStart=grep FIPS etc/crypto-policies/config [Install] -RequiredBy=fips-signal-ok.service -`, conf.Enable) +RequiredBy=fips-signal-ok.service` + +func testLiveFIPS(c cluster.TestCluster) { + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") + } + + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + config.AddSystemdUnit("fips-verify.service", fipsVerify, conf.Enable) config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) + keys, err := qc.Keys() + if err != nil { + c.Fatal(err) + } + config.CopyKeys(keys) + + overrideFW := func(builder *platform.QemuBuilder) error { + builder.Firmware = "uefi" + // This is the core change under test - adding the `fips=1` kernel argument via + // coreos-installer iso kargs modify should enter fips mode. + // Removing this line should cause this test to fail. + builder.AppendKernelArgs = "fips=1" + return nil + } - // Just for reliability, we'll run fully offline - builder.Append("-net", "none") + errchan := make(chan error) + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + output, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } - builder.SetConfig(config) - mach, err := builder.Exec() + // Read line in a goroutine and send errors to channel + go func() { + line, err := bufio.NewReader(output).ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", liveOKSignal) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line = strings.TrimSpace(line) + if line != liveOKSignal { + errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, liveOKSignal) + return + } + // OK! + errchan <- nil + }() + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + return builder.AddIso(isopath, "", false) + } + + callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callacks) if err != nil { - c.Fatal(err) + c.Fatalf("Unable to create test machine: %v", err) } - defer mach.Destroy() - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + err = <-errchan if err != nil { c.Fatal(err) } From 7c1604810ed0a30b380c3fb393a5b951ea4d7829 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 10:23:55 +0100 Subject: [PATCH 15/27] kola: refactor iso.*iscsi* tests --- mantle/kola/tests/iso/live-iscsi.go | 174 ++++++++++++++++------------ 1 file changed, 97 insertions(+), 77 deletions(-) diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 80229b9f68..759218bd35 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -1,11 +1,14 @@ package iso import ( + "bufio" _ "embed" + "fmt" "io" "os" "path/filepath" "strings" + "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" @@ -13,13 +16,20 @@ import ( "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/pkg/errors" ) +// Don't restrict networking for iSCSI tests; otherwise they fail. +func withInternet(t *register.Test) *register.Test { + t.Tags = append(t.Tags, "needs-internet") + return t +} + func init() { // Those currently work only on x86, see: https://github.com/coreos/fedora-coreos-tracker/issues/1657 - register.RegisterTest(isoTest("offline-install-iscsi.ibft.uefi", isoOfflineInstallIscsiIbftUefi, []string{"x86_64"})) - register.RegisterTest(isoTest("offline-install-iscsi.ibft-with-mpath", isoOfflineInstallIscsiIbftMpath, []string{"x86_64"})) - register.RegisterTest(isoTest("offline-install-iscsi.manual", isoOfflineInstallIscsiManual, []string{"x86_64"})) + register.RegisterTest(withInternet(isoTest("offline-install-iscsi.ibft.uefi", isoOfflineInstallIscsiIbftUefi, []string{"x86_64"}))) + register.RegisterTest(withInternet(isoTest("offline-install-iscsi.ibft-with-mpath", isoOfflineInstallIscsiIbftMpath, []string{"x86_64"}))) + register.RegisterTest(withInternet(isoTest("offline-install-iscsi.manual", isoOfflineInstallIscsiManual, []string{"x86_64"}))) } func isoOfflineInstallIscsiIbftUefi(c cluster.TestCluster) { @@ -81,16 +91,12 @@ var iscsi_butane_config string // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { c.Fatalf("Unsupported cluster type") } + // Prepare config var butane string if opts.enableIbft && opts.enableMultipath { butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") @@ -99,90 +105,104 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { } else if opts.manual { butane = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") } - - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) - } - builddir := kola.CosaBuild.Dir - isopath := filepath.Join(builddir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - c.Fatal(err) - } - defer builder.Close() - if err := builder.AddIso(isopath, "", false); err != nil { - c.Fatal(err) - } - - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) - } - - // Create a serial channel to read the logs from the nested VM - nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") - if err != nil { - c.Fatal(err) - } - - // Create a file to write the contents of the serial channel into - nestedVMConsole, err := os.OpenFile(filepath.Join(outdir, "nested_vm_console.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - c.Fatal(err) - } - - go func() { - _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) - if err != nil && err != io.EOF { - panic(err) - } - }() - - // empty disk to use as an iscsi target to install coreOS on and subseqently boot - // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers - err = builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}) - if err != nil { - c.Fatal(err) - } - - // We need more memory to start another VM within ! - builder.MemoryMiB = 2048 - var iscsiTargetConfig = conf.Butane(butane) - config, err := iscsiTargetConfig.Render(conf.FailWarnings) if err != nil { c.Fatal(err) } - err = forwardJournal(outdir, builder, config) + keys, err := qc.Keys() if err != nil { c.Fatal(err) } - + config.CopyKeys(keys) // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + config.MountHost("/var/cosaroot", true) - // enable network - builder.EnableUsermodeNetworking([]platform.HostForwardPort{}, "") - - // keep auto-login enabled for easier debug when running console - config.AddAutoLogin() + overrideFW := func(builder *platform.QemuBuilder) error { + if opts.enableUefi { + builder.Firmware = "uefi" + } + // We need more memory to start another VM within ! + builder.MemoryMiB = 2048 + return nil + } - builder.SetConfig(config) + errchan := make(chan error) + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + if err := builder.AddIso(isopath, "", false); err != nil { + return err + } - // Bind mount in the COSA rootfs into the VM so we can use it as a - // read-only rootfs for quickly starting the container to kola - // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml - builder.MountHost("/", "/var/cosaroot", true) - config.MountHost("/var/cosaroot", true) + output, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } - mach, err := builder.Exec() + // Create a serial channel to read the logs from the nested VM + nestedVmLogsChannel, err := builder.VirtioChannelRead("nestedvmlogs") + if err != nil { + return err + } + // Create a file to write the contents of the serial channel into + path := strings.Replace(builder.ConsoleFile, "/console.txt", "/nested_vm_console.txt", 1) + nestedVMConsole, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + go func() { + _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) + if err != nil && err != io.EOF { + panic(err) + } + }() + + // empty disk to use as an iscsi target to install coreOS on and subseqently boot + // Also add a 10G disk that we will mount on /var, to increase space available when pulling containers + if err := builder.AddDisksFromSpecs([]string{"10G:serial=target", "10G:serial=var"}); err != nil { + return err + } + // Bind mount in the COSA rootfs into the VM so we can use it as a + // read-only rootfs for quickly starting the container to kola + // qemuexec the nested VM for the test. See resources/iscsi_butane_setup.yaml + builder.MountHost("/", "/var/cosaroot", true) + + // Read line in a goroutine and send errors to channel + go func() { + exp := "iscsi-boot-ok" + line, err := bufio.NewReader(output).ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) + } else { + errchan <- errors.Wrapf(err, "reading from completion channel") + } + return + } + line = strings.TrimSpace(line) + if line != exp { + errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, exp) + return + } + // OK! + errchan <- nil + }() + + return nil + } + + callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callacks) if err != nil { - c.Fatal(err) + c.Fatalf("Unable to create test machine: %v", err) } - defer mach.Destroy() - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{"iscsi-boot-ok"}) + err = <-errchan if err != nil { c.Fatal(err) } From b6225575ccdff1ffd1792faa7b365946b5707f4d Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 11:04:34 +0100 Subject: [PATCH 16/27] kola: simplify iso.* tests by moving output validation into a separate function --- mantle/kola/tests/iso/common.go | 47 +++++++++++++++-------------- mantle/kola/tests/iso/live-fips.go | 26 +--------------- mantle/kola/tests/iso/live-iscsi.go | 25 +-------------- mantle/kola/tests/iso/live-login.go | 27 +---------------- 4 files changed, 27 insertions(+), 98 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 3459cd8260..918e6aa9b1 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -171,6 +171,29 @@ func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuil return builder, config, nil } +func CheckTestOutput(output *os.File, expected []string) error { + reader := bufio.NewReader(output) + for _, exp := range expected { + line, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + // this may be from QEMU getting killed or exiting; wait a bit + // to give a chance for .Wait() above to feed the channel with a + // better error + time.Sleep(1 * time.Second) + return fmt.Errorf("got EOF from completion channel, %s expected", exp) + } else { + return errors.Wrapf(err, "reading from completion channel") + } + } + line = strings.TrimSpace(line) + if line != exp { + return fmt.Errorf("unexpected string from completion channel: %q, expected: %q", line, exp) + } + } + return nil +} + func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { ctx := c.Context() @@ -212,29 +235,7 @@ func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") }() go func() { - r := bufio.NewReader(qchan) - for _, exp := range expected { - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line := strings.TrimSpace(l) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %s expected: %s", line, exp) - return - } - } - // OK! - errchan <- nil + errchan <- CheckTestOutput(qchan, expected) }() go func() { //check for error when switching boot order diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index 70f21c65ca..0361302476 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -1,12 +1,7 @@ package iso import ( - "bufio" - "fmt" - "io" "path/filepath" - "strings" - "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" @@ -80,26 +75,7 @@ func testLiveFIPS(c cluster.TestCluster) { // Read line in a goroutine and send errors to channel go func() { - line, err := bufio.NewReader(output).ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", liveOKSignal) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line = strings.TrimSpace(line) - if line != liveOKSignal { - errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, liveOKSignal) - return - } - // OK! - errchan <- nil + errchan <- CheckTestOutput(output, []string{liveOKSignal}) }() isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 759218bd35..26d40ddc8a 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -1,14 +1,11 @@ package iso import ( - "bufio" _ "embed" - "fmt" "io" "os" "path/filepath" "strings" - "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" @@ -170,27 +167,7 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { // Read line in a goroutine and send errors to channel go func() { - exp := "iscsi-boot-ok" - line, err := bufio.NewReader(output).ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line = strings.TrimSpace(line) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, exp) - return - } - // OK! - errchan <- nil + errchan <- CheckTestOutput(output, []string{"iscsi-boot-ok"}) }() return nil diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index e2cff5d51f..6e694dfc99 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -1,12 +1,7 @@ package iso import ( - "bufio" - "fmt" - "io" "path/filepath" - "strings" - "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/platform" @@ -52,27 +47,7 @@ version: 1.1.0`) // Read line in a goroutine and send errors to channel go func() { - exp := "coreos-liveiso-success" - line, err := bufio.NewReader(output).ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp) - } else { - errchan <- errors.Wrapf(err, "reading from completion channel") - } - return - } - line = strings.TrimSpace(line) - if line != exp { - errchan <- fmt.Errorf("Unexpected string from completion channel: %q, expected: %q", line, exp) - return - } - // OK! - errchan <- nil + errchan <- CheckTestOutput(output, []string{"coreos-liveiso-success"}) }() isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) From 519f028d9c333626bcdb9038c17da3907790f5b8 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 11:24:50 +0100 Subject: [PATCH 17/27] kola: remove redundant console and journal checks for iso.* tests These checks are already performed by harness.go:CheckConsole. --- mantle/kola/tests/iso/common.go | 39 --------------------------------- 1 file changed, 39 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 918e6aa9b1..9262b484e6 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -246,45 +246,6 @@ func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console } }() err := <-errchan - if err == nil { - // No error so far, check the console and journal files - consoleFile := filepath.Join(outdir, "console.txt") - journalFile := filepath.Join(outdir, "journal.txt") - files := []string{consoleFile, journalFile} - for _, file := range files { - fileName := filepath.Base(file) - // Check if the file exists - _, err := os.Stat(file) - if os.IsNotExist(err) { - fmt.Printf("The file: %v does not exist\n", fileName) - continue - } else if err != nil { - fmt.Println(err) - return err - } - // Read the contents of the file - fileContent, err := os.ReadFile(file) - if err != nil { - fmt.Println(err) - return err - } - // Check for badness with CheckConsole - warnOnly, badlines := kola.CheckConsole([]byte(fileContent), nil) - if len(badlines) > 0 { - for _, badline := range badlines { - if warnOnly { - c.Errorf("bad log line detected: %v", badline) - } else { - c.Logf("bad log line detected: %v", badline) - } - } - if !warnOnly { - err = fmt.Errorf("errors found in log files") - return err - } - } - } - } return err } From b5d4b4976011ac21e9db7cbf4efc930b8a0bc0b3 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 24 Nov 2025 12:13:20 +0100 Subject: [PATCH 18/27] kola: refactor iso.as-disk* tests --- mantle/kola/tests/iso/live-as-disk.go | 69 ++++++++++++++------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index bd1ac2015a..6eb47099e2 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -6,8 +6,10 @@ import ( "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" + "github.com/pkg/errors" ) func init() { @@ -16,7 +18,6 @@ func init() { register.RegisterTest(isoTest("as-disk", isoAsDisk, []string{"x86_64"})) register.RegisterTest(isoTest("as-disk.uefi", isoAsDiskUefi, []string{"x86_64"})) register.RegisterTest(isoTest("as-disk.uefi-secure", isoAsDiskUefiSecure, []string{"x86_64"})) - register.RegisterTest(isoTest("as-disk.4k.uefi", isoAsDisk4kUefi, []string{"x86_64"})) } func isoAsDisk(c cluster.TestCluster) { @@ -40,53 +41,57 @@ func isoAsDiskUefiSecure(c cluster.TestCluster) { isoTestAsDisk(c, opts) } -func isoAsDisk4kUefi(c cluster.TestCluster) { - opts := IsoTestOpts{ - enable4k: true, - enableUefi: true, - } - opts.SetInsecureOnDevBuild() - isoTestAsDisk(c, opts) -} - func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - //var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - //qc = pc - default: + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { c.Fatalf("Unsupported cluster type") } - isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - builder, config, err := newQemuBuilder(opts, outdir) + config, err := conf.EmptyIgnition().Render(conf.FailWarnings) if err != nil { c.Fatal(err) } - defer builder.Close() - // Drop the bootindex bit (applicable to all arches except s390x and ppc64le); we want it to be the default - if err := builder.AddIso(isopath, "", true); err != nil { + config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) + config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) + keys, err := qc.Keys() + if err != nil { c.Fatal(err) } + config.CopyKeys(keys) - completionChannel, err := builder.VirtioChannelRead("testisocompletion") - if err != nil { - c.Fatal(err) + overrideFW := func(builder *platform.QemuBuilder) error { + switch { + case opts.enableUefiSecure: + builder.Firmware = "uefi-secure" + case opts.enableUefi: + builder.Firmware = "uefi" + } + return nil } - config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) - config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - builder.SetConfig(config) + errchan := make(chan error) + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + output, err := builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up virtio-serial channel") + } - mach, err := builder.Exec() + // Read line in a goroutine and send errors to channel + go func() { + errchan <- CheckTestOutput(output, []string{liveOKSignal}) + }() + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + return builder.AddIso(isopath, "", true) + } + + callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callacks) if err != nil { - c.Fatal(err) + c.Fatalf("Unable to create test machine: %v", err) } - defer mach.Destroy() - err = awaitCompletion(c, mach, opts.console, outdir, completionChannel, nil, []string{liveOKSignal}) + err = <-errchan if err != nil { c.Fatal(err) } From 46d2763ca4b5582f4f70ef0ff2e0716dc0244011 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Wed, 26 Nov 2025 08:38:37 +0100 Subject: [PATCH 19/27] kola: refactor iso.*-install* tests --- mantle/kola/tests/iso/common.go | 30 +- mantle/kola/tests/iso/live-iso.go | 421 +++++++++++++++++++----- mantle/platform/machine/qemu/cluster.go | 35 +- mantle/platform/machine/qemu/metal.go | 261 --------------- 4 files changed, 398 insertions(+), 349 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 9262b484e6..cc1bb727c9 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -22,8 +22,6 @@ import ( const ( installTimeoutMins = 12 - // https://github.com/coreos/fedora-coreos-config/pull/2544 - liveISOFromRAMKarg = "coreos.liveiso.fromram" ) type IsoTestOpts struct { @@ -427,3 +425,31 @@ ExecStart=/usr/bin/nmcli c show br-ex RequiredBy=coreos-installer.target # for target system RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) + +var bootStartedSignal = "boot-started-OK" +var bootStartedUnit = fmt.Sprintf(`[Unit] +Description=TestISO Boot Started +Requires=dev-virtio\\x2dports-bootstarted.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/bootstarted' +[Install] +RequiredBy=coreos-installer.target`, bootStartedSignal) + +var coreosInstallerMultipathUnit = `[Unit] +Description=TestISO Enable Multipath +Before=multipathd.service +DefaultDependencies=no +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/mpathconf --enable +[Install] +WantedBy=coreos-installer.target` + +var waitForMpathTargetConf = `[Unit] +Requires=dev-mapper-mpatha.device +After=dev-mapper-mpatha.device` diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index c374262979..165c6ede0c 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -3,37 +3,44 @@ package iso import ( _ "embed" "fmt" + "net" + "net/http" "os" + "os/exec" + "path/filepath" "strings" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) func init() { - register.RegisterTest(isoTest("install", isoInstall, []string{"x86_64"})) + register.RegisterTest(withInternet(isoTest("install", isoInstall, []string{"x86_64"}))) register.RegisterTest(isoTest("offline-install", isoOfflineInstall, []string{"x86_64", "s390x", "ppc64le"})) register.RegisterTest(isoTest("offline-install.uefi", isoOfflineInstallUefi, []string{"aarch64"})) register.RegisterTest(isoTest("offline-install.4k", isoOfflineInstall4k, []string{"s390x"})) register.RegisterTest(isoTest("offline-install.mpath", isoOfflineInstallMpath, []string{"x86_64", "s390x", "ppc64le"})) register.RegisterTest(isoTest("offline-install.mpath.uefi", isoOfflineInstallMpathUefi, []string{"aarch64"})) - register.RegisterTest(isoTest("offline-install-fromram.4k", isoOfflineInstallFromRam4k, []string{"ppc64le"})) register.RegisterTest(isoTest("offline-install-fromram.4k.uefi", isoOfflineInstallFromRam4kUefi, []string{"x86_64", "aarch64"})) - register.RegisterTest(isoTest("miniso-install", isoMinisoInstall, []string{"x86_64", "s390x", "ppc64le"})) - register.RegisterTest(isoTest("miniso-install.uefi", isoMinisoInstallUefi, []string{"aarch64"})) - register.RegisterTest(isoTest("miniso-install.4k", isoMinisoInstall4k, []string{"ppc64le"})) - register.RegisterTest(isoTest("miniso-install.4k.uefi", isoMinisoInstall4kUefi, []string{"x86_64", "aarch64"})) - - register.RegisterTest(isoTest("miniso-install.nm", isoMinisoInstallNm, []string{"x86_64", "s390x", "ppc64le"})) - register.RegisterTest(isoTest("miniso-install.nm.uefi", isoMinisoInstallNmUefi, []string{"aarch64"})) - register.RegisterTest(isoTest("miniso-install.4k.nm", isoMinisoInstall4kNm, []string{"ppc64le", "s390x"})) - register.RegisterTest(isoTest("miniso-install.4k.nm.uefi", isoMinisoInstall4kNmUefi, []string{"x86_64", "aarch64"})) + + register.RegisterTest(withInternet(isoTest("miniso-install", isoMinisoInstall, []string{"x86_64", "s390x", "ppc64le"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.uefi", isoMinisoInstallUefi, []string{"aarch64"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k", isoMinisoInstall4k, []string{"ppc64le"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k.uefi", isoMinisoInstall4kUefi, []string{"x86_64", "aarch64"}))) + + register.RegisterTest(withInternet(isoTest("miniso-install.nm", isoMinisoInstallNm, []string{"x86_64", "s390x", "ppc64le"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.nm.uefi", isoMinisoInstallNmUefi, []string{"aarch64"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k.nm", isoMinisoInstall4kNm, []string{"ppc64le", "s390x"}))) + register.RegisterTest(withInternet(isoTest("miniso-install.4k.nm.uefi", isoMinisoInstall4kNmUefi, []string{"x86_64", "aarch64"}))) } func isoMinisoInstall(c cluster.TestCluster) { @@ -41,7 +48,7 @@ func isoMinisoInstall(c cluster.TestCluster) { isMiniso: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoMinisoInstallUefi(c cluster.TestCluster) { @@ -50,7 +57,7 @@ func isoMinisoInstallUefi(c cluster.TestCluster) { enableUefi: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoMinisoInstall4k(c cluster.TestCluster) { @@ -59,7 +66,7 @@ func isoMinisoInstall4k(c cluster.TestCluster) { isMiniso: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoMinisoInstall4kUefi(c cluster.TestCluster) { @@ -69,7 +76,7 @@ func isoMinisoInstall4kUefi(c cluster.TestCluster) { isMiniso: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoMinisoInstallNm(c cluster.TestCluster) { @@ -78,7 +85,7 @@ func isoMinisoInstallNm(c cluster.TestCluster) { isMiniso: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoMinisoInstallNmUefi(c cluster.TestCluster) { @@ -88,7 +95,7 @@ func isoMinisoInstallNmUefi(c cluster.TestCluster) { enableUefi: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoMinisoInstall4kNm(c cluster.TestCluster) { @@ -98,7 +105,7 @@ func isoMinisoInstall4kNm(c cluster.TestCluster) { isMiniso: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoMinisoInstall4kNmUefi(c cluster.TestCluster) { @@ -109,13 +116,13 @@ func isoMinisoInstall4kNmUefi(c cluster.TestCluster) { isMiniso: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoInstall(c cluster.TestCluster) { opts := IsoTestOpts{} opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoOfflineInstall(c cluster.TestCluster) { @@ -123,7 +130,7 @@ func isoOfflineInstall(c cluster.TestCluster) { isOffline: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoOfflineInstallUefi(c cluster.TestCluster) { @@ -132,7 +139,7 @@ func isoOfflineInstallUefi(c cluster.TestCluster) { isOffline: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoOfflineInstall4k(c cluster.TestCluster) { @@ -141,7 +148,7 @@ func isoOfflineInstall4k(c cluster.TestCluster) { isOffline: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoOfflineInstallMpath(c cluster.TestCluster) { @@ -150,7 +157,7 @@ func isoOfflineInstallMpath(c cluster.TestCluster) { isOffline: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoOfflineInstallMpathUefi(c cluster.TestCluster) { @@ -160,7 +167,7 @@ func isoOfflineInstallMpathUefi(c cluster.TestCluster) { isOffline: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoOfflineInstallFromRam4k(c cluster.TestCluster) { @@ -170,7 +177,7 @@ func isoOfflineInstallFromRam4k(c cluster.TestCluster) { isISOFromRAM: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { @@ -181,95 +188,355 @@ func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { isISOFromRAM: true, } opts.SetInsecureOnDevBuild() - isoLiveIso(c, opts) + isoLiveInstall(c, opts) } -func isoLiveIso(c cluster.TestCluster, opts IsoTestOpts) { - var outdir string - var qc *qemu.Cluster - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - qc = pc - default: - c.Fatalf("Unsupported cluster type") +// Sometimes the logs that stream from various virtio streams can be +// incomplete because they depend on services inside the guest. +// When you are debugging earlyboot/initramfs issues this can be +// problematic. Let's add a hook here to enable more debugging. +func renderCosaTestIsoDebugKargs() []string { + if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { + return []string{"systemd.log_color=0", "systemd.log_level=debug", + "systemd.journald.forward_to_console=1", + "systemd.journald.max_level_console=debug"} + } else { + return []string{} } +} + +// This object gets serialized to YAML and fed to coreos-installer: +// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format +type coreosInstallerConfig struct { + ImageURL string `yaml:"image-url,omitempty"` + IgnitionFile string `yaml:"ignition-file,omitempty"` + Insecure bool `yaml:"insecure,omitempty"` + AppendKargs []string `yaml:"append-karg,omitempty"` + CopyNetwork bool `yaml:"copy-network,omitempty"` + DestDevice string `yaml:"dest-device,omitempty"` + Console []string `yaml:"console,omitempty"` +} + +// defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option +const defaultQemuHostIPv4 = "10.0.2.2" - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) +func isoLiveInstall(c cluster.TestCluster, opts IsoTestOpts) { + if opts.isMiniso && opts.isOffline { // ideally this'd be one enum parameter + c.Fatal("Can't run minimal install offline") + } + if opts.isOffline && opts.addNmKeyfile { + c.Fatal("Cannot use `--add-nm-keyfile` with offline mode") } - inst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - Insecure: opts.instInsecure, - Native4k: opts.enable4k, - MultiPathDisk: opts.enableMultipath, + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") + } + if opts.enable4k { + qc.EnforeNative4k() + } + if opts.enableMultipath { + qc.EnforeMultipath() } - tmpd, err := os.MkdirTemp("", "kola-iso.live") + tempdir, err := os.MkdirTemp("/var/tmp", "iso") if err != nil { c.Fatal(err) } - defer os.RemoveAll(tmpd) + defer func() { + os.RemoveAll(tempdir) + }() - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) - if err != nil { + if err := isoRunTest(qc, opts, tempdir); err != nil { c.Fatal(err) } +} - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) +func isoRunTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { + keys, err := qc.Keys() if err != nil { - c.Fatal(err) + return err } - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) if err != nil { - c.Fatal(err) + return err + } + targetConfig.CopyKeys(keys) + targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) + targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) + if opts.enableMultipath { + targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) + } + if opts.addNmKeyfile { + targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) + } + + isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) + + installerConfig := coreosInstallerConfig{ + IgnitionFile: "/var/opt/pointer.ign", + DestDevice: "/dev/vda", + AppendKargs: renderCosaTestIsoDebugKargs(), + Insecure: opts.instInsecure, + CopyNetwork: opts.addNmKeyfile, // force networking on in the initrd to verify the keyfile was used + } + + var serializedTargetConfig string + if opts.isOffline { + // note we leave ImageURL empty here; offline installs should now be the + // default! + + // we want to test that a full offline install works; that includes the + // final installed host booting offline + serializedTargetConfig = targetConfig.String() + } else { + listener, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + port := listener.Addr().(*net.TCPAddr).Port + baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) + + // This is subtle but: for the minimal case, while we need networking to fetch the + // rootfs, the primary install flow will still rely on osmet. So let's keep ImageURL + // empty to exercise that path. In the future, this could be a separate scenario + // (likely we should drop the "offline" naming and have a "remote" tag on the + // opposite scenarios instead which fetch the metal image, so then we'd have + // "[min]iso-install" and "[min]iso-remote-install"). + if opts.isMiniso { + isopath, err = createMiniso(tempdir, isopath, baseurl) + if err != nil { + return err + } + } else { + var metalimg string + if opts.enable4k { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path + } else { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal.Path + } + metalname, err := setupMetalImage(kola.CosaBuild.Dir, metalimg, tempdir) + if err != nil { + return err + } + installerConfig.ImageURL = fmt.Sprintf("%s/%s", baseurl, metalname) + } + + if opts.addNmKeyfile { + nmKeyfiles := make(map[string]string) + nmKeyfiles[nmConnectionFile] = nmConnection + if err := embedNmkeyfiles(tempdir, nmKeyfiles, isopath); err != nil { + return err + } + } + + // In this case; the target config is jut a tiny wrapper that wants to + // fetch our hosted target.ign config + // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 + // when it arrives + if err := targetConfig.WriteFile(filepath.Join(tempdir, "target.ign")); err != nil { + return err + } + targetConfig, err = conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return err + } + targetConfig.AddConfigSource(baseurl + "/target.ign") + serializedTargetConfig = targetConfig.String() + + //nolint // Yeah this leaks + go func() { + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tempdir))) + http.Serve(listener, mux) + }() + } + + // XXX: https://github.com/coreos/coreos-installer/issues/1171 + if coreosarch.CurrentRpmArch() != "s390x" { + installerConfig.Console = []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]} + } + if opts.enableMultipath { + // we only have one multipath device so it has to be that + installerConfig.DestDevice = "/dev/mapper/mpatha" + installerConfig.AppendKargs = append(installerConfig.AppendKargs, "rd.multipath=default", "root=/dev/disk/by-label/dm-mpath-root", "rw") } - var isoKernelArgs []string - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) + installerConfigData, err := yaml.Marshal(installerConfig) + if err != nil { + return err + } + mode := 0644 - liveConfig := *virtioJournalConfig + liveConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + return err + } + liveConfig.CopyKeys(keys) liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) volumeIdUnitContents := fmt.Sprintf(verifyIsoVolumeId, kola.CosaBuild.Meta.Name) liveConfig.AddSystemdUnit("verify-iso-volume-id.service", volumeIdUnitContents, conf.Enable) - - targetConfig := *virtioJournalConfig - targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - if inst.MultiPathDisk { - targetConfig.AddSystemdUnit("coreos-test-installer-multipathed.service", multipathedRoot, conf.Enable) + liveConfig.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) + liveConfig.AddFile(installerConfig.IgnitionFile, serializedTargetConfig, mode) + liveConfig.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) + liveConfig.AddAutoLogin() + if opts.enableMultipath { + liveConfig.AddSystemdUnit("coreos-installer-multipath.service", coreosInstallerMultipathUnit, conf.Enable) + liveConfig.AddSystemdUnitDropin("coreos-installer.service", "wait-for-mpath-target.conf", waitForMpathTargetConf) } - if opts.addNmKeyfile { liveConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - targetConfig.AddSystemdUnit("coreos-test-nm-keyfile.service", verifyNmKeyfile, conf.Enable) - // NM keyfile via `iso network embed` - inst.NmKeyfiles[nmConnectionFile] = nmConnection // nmstate config via live Ignition config, propagated via // --copy-network, which is enabled by inst.NmKeyfiles liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644) } - if opts.isISOFromRAM { - isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg) + overrideFW := func(builder *platform.QemuBuilder) error { + builder.MemoryMiB = 4096 + if opts.enableUefi { + builder.Firmware = "uefi" + } + kargs := renderCosaTestIsoDebugKargs() + if opts.isISOFromRAM { + // https://github.com/coreos/fedora-coreos-config/pull/2544 + kargs = append(kargs, "coreos.liveiso.fromram") + } + if opts.addNmKeyfile { + kargs = append(kargs, "rd.neednet=1") + } + builder.AppendKernelArgs = strings.Join(kargs, " ") + return nil + } + + setupNet := func(o platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + if !opts.isOffline { + // also save pointer config into the output dir for debugging + path := filepath.Join(qc.RuntimeConf().OutputDir, builder.UUID, "config-target-pointer.ign") + if err := targetConfig.WriteFile(path); err != nil { + return err + } + return qc.SetupDefaultNetwork(o, builder) + } + return nil + } + + var isoCompletionOutput *os.File + var bootStartedOutput *os.File + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return err + } + } + isoCompletionOutput, err = builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up testisocompletion virtio-serial channel") + } + bootStartedOutput, err = builder.VirtioChannelRead("bootstarted") + if err != nil { + return errors.Wrap(err, "setting up bootstarted virtio-serial channel") + } + return builder.AddIso(isopath, "bootindex=3", false) } - mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, opts.isOffline, opts.isMiniso) + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true + callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} + qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callacks) if err != nil { - c.Fatal(err) + return errors.Wrap(err, "unable to create test machine") } - qc.AddMach(mach) - err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + + errchan := make(chan error) + go func() { + errchan <- CheckTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) + }() + + //check for error when switching boot order + go func() { + if err := CheckTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { + errchan <- err + return + } + if err := qc.Instance(qm).SwitchBootOrder(); err != nil { + errchan <- errors.Wrapf(err, "switching boot order failed") + return + } + }() + + err = <-errchan + return err +} + +func absSymlink(src, dest string) error { + src, err := filepath.Abs(src) if err != nil { - c.Fatal(err) + return err + } + return os.Symlink(src, dest) +} + +// setupMetalImage creates a symlink to the metal image. +func setupMetalImage(builddir, metalimg, destdir string) (string, error) { + if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { + return "", err + } + return metalimg, nil +} + +func createMiniso(tempd string, isopath string, url string) (string, error) { + minisopath := filepath.Join(tempd, "minimal.iso") + // This is obviously also available in the build dir, but to be realistic, + // let's take it from --rootfs-output + rootfs_path := filepath.Join(tempd, "rootfs.img") + // Ideally we'd use the coreos-installer of the target build here, because it's part + // of the test workflow, but that's complex... Sadly, probably easiest is to spin up + // a VM just to get the minimal ISO. + cmd := exec.Command("coreos-installer", "iso", "extract", "minimal-iso", isopath, + minisopath, "--output-rootfs", rootfs_path, "--rootfs-url", url+"/rootfs.img") + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return "", errors.Wrapf(err, "running coreos-installer iso extract minimal") + } + return minisopath, nil +} + +func embedNmkeyfiles(tempd string, nmKeyfiles map[string]string, isopath string) error { + var keyfileArgs []string + for nmName, nmContents := range nmKeyfiles { + path := filepath.Join(tempd, nmName) + if err := os.WriteFile(path, []byte(nmContents), 0600); err != nil { + return err + } + keyfileArgs = append(keyfileArgs, "--keyfile", path) + } + if len(keyfileArgs) > 0 { + args := []string{"iso", "network", "embed", isopath} + args = append(args, keyfileArgs...) + cmd := exec.Command("coreos-installer", args...) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "running coreos-installer iso network embed") + } } + return nil } diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index ad06978a39..590667b2a8 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -51,6 +51,14 @@ type BuilderCallbacks struct { OverrideDefaults func(builder *platform.QemuBuilder) error } +func (qc *Cluster) EnforeNative4k() { + qc.flight.opts.Native4k = true +} + +func (qc *Cluster) EnforeMultipath() { + qc.flight.opts.MultiPathDisk = true +} + func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) { return qc.NewMachineWithOptions(userdata, platform.MachineOptions{}) } @@ -153,18 +161,19 @@ func (qc *Cluster) NewMachineWithQemuOptionsAndBuilderCallbacks(userdata any, op } qm.inst = inst - err = util.Retry(6, 5*time.Second, func() error { - var err error - qm.ip, err = inst.SSHAddress() + if builder.UsermodeNetworking { + err = util.Retry(6, 5*time.Second, func() error { + var err error + qm.ip, err = inst.SSHAddress() + if err != nil { + return err + } + return nil + }) if err != nil { - return err + return nil, err } - return nil - }) - if err != nil { - return nil, err } - // Run StartMachine, which blocks on the machine being booted up enough // for SSH access, but only if the caller didn't tell us not to. if !options.SkipStartMachine { @@ -189,6 +198,14 @@ func (qc *Cluster) NewMachineWithQemuOptionsAndBuilderCallbacks(userdata any, op return qm, nil } +func (qc *Cluster) Instance(m platform.Machine) *platform.QemuInstance { + switch pm := m.(type) { + case *machine: + return pm.inst + default: + return nil + } +} func (qc *Cluster) Destroy() { qc.tearingDown = true diff --git a/mantle/platform/machine/qemu/metal.go b/mantle/platform/machine/qemu/metal.go index d49e489bae..d4730bfe87 100644 --- a/mantle/platform/machine/qemu/metal.go +++ b/mantle/platform/machine/qemu/metal.go @@ -574,264 +574,3 @@ type installerConfig struct { DestDevice string `yaml:"dest-device,omitempty"` Console []string `yaml:"console,omitempty"` } - -func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgnition conf.Conf, outdir string, offline, minimal bool) (*machine, error) { - artifacts := []string{"live-iso"} - if !offline { - if inst.Native4k { - artifacts = append(artifacts, "metal4k") - } else { - artifacts = append(artifacts, "metal") - } - } - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - if minimal && offline { // ideally this'd be one enum parameter - panic("Can't run minimal install offline") - } - if offline && len(inst.NmKeyfiles) > 0 { - return nil, fmt.Errorf("Cannot use `--add-nm-keyfile` with offline mode") - } - - installerConfig := installerConfig{ - IgnitionFile: "/var/opt/pointer.ign", - DestDevice: "/dev/vda", - AppendKargs: renderCosaTestIsoDebugKargs(), - } - - // XXX: https://github.com/coreos/coreos-installer/issues/1171 - if coreosarch.CurrentRpmArch() != "s390x" { - installerConfig.Console = []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]} - } - - if inst.MultiPathDisk { - // we only have one multipath device so it has to be that - installerConfig.DestDevice = "/dev/mapper/mpatha" - installerConfig.AppendKargs = append(installerConfig.AppendKargs, "rd.multipath=default", "root=/dev/disk/by-label/dm-mpath-root", "rw") - } - - inst.kargs = append(renderCosaTestIsoDebugKargs(), kargs...) - inst.ignition = targetIgnition - inst.liveIgnition = liveIgnition - - tempdir, err := os.MkdirTemp("/var/tmp", "mantle-metal") - if err != nil { - return nil, err - } - cleanupTempdir := true - defer func() { - if cleanupTempdir { - os.RemoveAll(tempdir) - } - }() - - if err := inst.ignition.WriteFile(filepath.Join(tempdir, "target.ign")); err != nil { - return nil, err - } - // and write it once more in the output dir for debugging - if err := inst.ignition.WriteFile(filepath.Join(outdir, "config-target.ign")); err != nil { - return nil, err - } - - builddir := inst.CosaBuild.Dir - srcisopath := filepath.Join(builddir, inst.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - - // Copy the ISO to a new location for modification. - // This is a bit awkward; we copy here, but QemuBuilder will also copy - // again (in `setupIso()`). I didn't want to lower the NM keyfile stuff - // into QemuBuilder. And plus, both tempdirs should be in /var/tmp so - // the `cp --reflink=auto` that QemuBuilder does should just reflink. - newIso := filepath.Join(tempdir, "install.iso") - cmd := exec.Command("cp", "--reflink=auto", srcisopath, newIso) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "copying iso") - } - // Make it writable so we can modify it - if err := os.Chmod(newIso, 0644); err != nil { - return nil, errors.Wrapf(err, "setting permissions on iso") - } - srcisopath = newIso - - var serializedTargetConfig string - if offline { - // note we leave ImageURL empty here; offline installs should now be the - // default! - - // we want to test that a full offline install works; that includes the - // final installed host booting offline - serializedTargetConfig = inst.ignition.String() - } else { - var metalimg string - if inst.Native4k { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path - } else { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal.Path - } - metalname, err := setupMetalImage(builddir, metalimg, tempdir) - if err != nil { - return nil, errors.Wrapf(err, "setting up metal image") - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tempdir))) - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - port := listener.Addr().(*net.TCPAddr).Port - //nolint // Yeah this leaks - go func() { - http.Serve(listener, mux) - }() - baseurl := fmt.Sprintf("http://%s:%d", defaultQemuHostIPv4, port) - - // This is subtle but: for the minimal case, while we need networking to fetch the - // rootfs, the primary install flow will still rely on osmet. So let's keep ImageURL - // empty to exercise that path. In the future, this could be a separate scenario - // (likely we should drop the "offline" naming and have a "remote" tag on the - // opposite scenarios instead which fetch the metal image, so then we'd have - // "[min]iso-install" and "[min]iso-remote-install"). - if !minimal { - installerConfig.ImageURL = fmt.Sprintf("%s/%s", baseurl, metalname) - } - - if minimal { - minisopath := filepath.Join(tempdir, "minimal.iso") - // This is obviously also available in the build dir, but to be realistic, - // let's take it from --rootfs-output - rootfs_path := filepath.Join(tempdir, "rootfs.img") - // Ideally we'd use the coreos-installer of the target build here, because it's part - // of the test workflow, but that's complex... Sadly, probably easiest is to spin up - // a VM just to get the minimal ISO. - cmd := exec.Command("coreos-installer", "iso", "extract", "minimal-iso", srcisopath, - minisopath, "--output-rootfs", rootfs_path, "--rootfs-url", baseurl+"/rootfs.img") - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso extract minimal") - } - srcisopath = minisopath - } - - // In this case; the target config is jut a tiny wrapper that wants to - // fetch our hosted target.ign config - - // TODO also use https://github.com/coreos/coreos-installer/issues/118#issuecomment-585572952 - // when it arrives - targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, err - } - targetConfig.AddConfigSource(baseurl + "/target.ign") - serializedTargetConfig = targetConfig.String() - - // also save pointer config into the output dir for debugging - if err := targetConfig.WriteFile(filepath.Join(outdir, "config-target-pointer.ign")); err != nil { - return nil, err - } - } - - var keyfileArgs []string - for nmName, nmContents := range inst.NmKeyfiles { - path := filepath.Join(tempdir, nmName) - if err := os.WriteFile(path, []byte(nmContents), 0600); err != nil { - return nil, err - } - keyfileArgs = append(keyfileArgs, "--keyfile", path) - } - if len(keyfileArgs) > 0 { - - args := []string{"iso", "network", "embed", srcisopath} - args = append(args, keyfileArgs...) - cmd = exec.Command("coreos-installer", args...) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso network embed") - } - - installerConfig.CopyNetwork = true - - // force networking on in the initrd to verify the keyfile was used - inst.kargs = append(inst.kargs, "rd.neednet=1") - } - - if len(inst.kargs) > 0 { - args := []string{"iso", "kargs", "modify", srcisopath} - for _, karg := range inst.kargs { - args = append(args, "--append", karg) - } - cmd = exec.Command("coreos-installer", args...) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return nil, errors.Wrapf(err, "running coreos-installer iso kargs") - } - } - - if inst.Insecure { - installerConfig.Insecure = true - } - - installerConfigData, err := yaml.Marshal(installerConfig) - if err != nil { - return nil, err - } - mode := 0644 - - inst.liveIgnition.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) - inst.liveIgnition.AddFile(installerConfig.IgnitionFile, serializedTargetConfig, mode) - inst.liveIgnition.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) - inst.liveIgnition.AddAutoLogin() - - if inst.MultiPathDisk { - inst.liveIgnition.AddSystemdUnit("coreos-installer-multipath.service", `[Unit] -Description=TestISO Enable Multipath -Before=multipathd.service -DefaultDependencies=no -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/usr/sbin/mpathconf --enable -[Install] -WantedBy=coreos-installer.target`, conf.Enable) - inst.liveIgnition.AddSystemdUnitDropin("coreos-installer.service", "wait-for-mpath-target.conf", `[Unit] -Requires=dev-mapper-mpatha.device -After=dev-mapper-mpatha.device`) - } - - qemubuilder := inst.Builder - bootStartedChan, err := qemubuilder.VirtioChannelRead("bootstarted") - if err != nil { - return nil, err - } - - qemubuilder.SetConfig(&inst.liveIgnition) - - // also save live config into the output dir for debugging - liveConfigPath := filepath.Join(outdir, "config-live.ign") - if err := inst.liveIgnition.WriteFile(liveConfigPath); err != nil { - return nil, err - } - - if err := qemubuilder.AddIso(srcisopath, "bootindex=3", false); err != nil { - return nil, err - } - - // With the recent change to use qemu -nodefaults (bc68d7c) we need to - // request network. Otherwise we get no network devices. - if !offline { - qemubuilder.UsermodeNetworking = true - } - - qinst, err := qemubuilder.Exec() - if err != nil { - return nil, err - } - cleanupTempdir = false // Transfer ownership - instmachine := machine{ - inst: qinst, - tempdir: tempdir, - } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) - return &instmachine, nil -} From e28bc117ea48312bc1904f557fc6b5fbae935d7b Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 27 Nov 2025 10:28:20 +0100 Subject: [PATCH 20/27] kola: small fixes for iso.* tests --- mantle/kola/tests/iso/common.go | 2 +- mantle/kola/tests/iso/live-iscsi.go | 5 +++-- mantle/platform/machine/qemu/metal.go | 7 +------ 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index cc1bb727c9..58391afc03 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -129,7 +129,7 @@ func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.C go func() { _, err := io.Copy(journalOut, journalPipe) if err != nil && err != io.EOF { - panic(err) + fmt.Printf("error copying journal: %v\n", err) } }() diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 26d40ddc8a..0ab472537b 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -2,6 +2,7 @@ package iso import ( _ "embed" + "fmt" "io" "os" "path/filepath" @@ -143,7 +144,7 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { return err } // Create a file to write the contents of the serial channel into - path := strings.Replace(builder.ConsoleFile, "/console.txt", "/nested_vm_console.txt", 1) + path := filepath.Join(filepath.Dir(builder.ConsoleFile), "nested_vm_console.txt") nestedVMConsole, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { return err @@ -151,7 +152,7 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { go func() { _, err := io.Copy(nestedVMConsole, nestedVmLogsChannel) if err != nil && err != io.EOF { - panic(err) + fmt.Printf("error copying nested VM logs: %v\n", err) } }() diff --git a/mantle/platform/machine/qemu/metal.go b/mantle/platform/machine/qemu/metal.go index d4730bfe87..f5d95ed345 100644 --- a/mantle/platform/machine/qemu/metal.go +++ b/mantle/platform/machine/qemu/metal.go @@ -35,12 +35,7 @@ import ( "github.com/coreos/coreos-assembler/mantle/util" ) -const ( - // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option - defaultQemuHostIPv4 = "10.0.2.2" - - bootStartedSignal = "boot-started-OK" -) +const bootStartedSignal = "boot-started-OK" // TODO derive this from docs, or perhaps include kargs in cosa metadata? var baseKargs = []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} From 70627267a38347ffa0990de898882dcec9c258fc Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Thu, 27 Nov 2025 10:44:02 +0100 Subject: [PATCH 21/27] kola: don't fail iso.* tests when live arifacts don't exist --- mantle/kola/tests/iso/common.go | 10 ++++++++++ mantle/kola/tests/iso/live-as-disk.go | 6 ++++++ mantle/kola/tests/iso/live-fips.go | 6 ++++++ mantle/kola/tests/iso/live-iscsi.go | 5 +++++ mantle/kola/tests/iso/live-iso.go | 4 ++++ mantle/kola/tests/iso/live-login.go | 7 ++++--- mantle/kola/tests/iso/live-pxe.go | 5 +++++ 7 files changed, 40 insertions(+), 3 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 58391afc03..73c518f6ff 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -192,6 +192,16 @@ func CheckTestOutput(output *os.File, expected []string) error { return nil } +func EnsureLiveArtifactsExist() error { + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + return errors.Errorf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) + } + if kola.CosaBuild.Meta.BuildArtifacts.Metal == nil || kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative == nil { + return errors.Errorf("Build %s is missing live metal artifacts\n", kola.CosaBuild.Meta.Name) + } + return nil +} + func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { ctx := c.Context() diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index 6eb47099e2..2895742b0e 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -1,6 +1,7 @@ package iso import ( + "fmt" "path/filepath" "github.com/coreos/coreos-assembler/mantle/kola" @@ -42,6 +43,11 @@ func isoAsDiskUefiSecure(c cluster.TestCluster) { } func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { + if err := EnsureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + qc, ok := c.Cluster.(*qemu.Cluster) if !ok { c.Fatalf("Unsupported cluster type") diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index 0361302476..6a06f6d3e4 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -1,6 +1,7 @@ package iso import ( + "fmt" "path/filepath" "github.com/coreos/coreos-assembler/mantle/kola" @@ -39,6 +40,11 @@ ExecStart=grep FIPS etc/crypto-policies/config RequiredBy=fips-signal-ok.service` func testLiveFIPS(c cluster.TestCluster) { + if err := EnsureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + qc, ok := c.Cluster.(*qemu.Cluster) if !ok { c.Fatalf("Unsupported cluster type") diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 0ab472537b..3c5632cb86 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -89,6 +89,11 @@ var iscsi_butane_config string // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { + if err := EnsureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + qc, ok := c.Cluster.(*qemu.Cluster) if !ok { c.Fatalf("Unsupported cluster type") diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 165c6ede0c..4ecc8ad857 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -221,6 +221,10 @@ type coreosInstallerConfig struct { const defaultQemuHostIPv4 = "10.0.2.2" func isoLiveInstall(c cluster.TestCluster, opts IsoTestOpts) { + if err := EnsureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } if opts.isMiniso && opts.isOffline { // ideally this'd be one enum parameter c.Fatal("Can't run minimal install offline") } diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index 6e694dfc99..0a81c2dbad 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -1,6 +1,7 @@ package iso import ( + "fmt" "path/filepath" "github.com/coreos/coreos-assembler/mantle/kola" @@ -20,10 +21,10 @@ func init() { } func testLiveLogin(c cluster.TestCluster, enableUefi bool, enableUefiSecure bool) { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - c.Fatalf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) + if err := EnsureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return } - butane := conf.Butane(` variant: fcos version: 1.1.0`) diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 0057becb55..95ac56a70d 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -130,6 +130,11 @@ RequiredBy=coreos-installer.target ` func testPXE(c cluster.TestCluster, opts IsoTestOpts) { + if err := EnsureLiveArtifactsExist(); err != nil { + fmt.Println(err) + return + } + var outdir string var qc *qemu.Cluster From 18f3e3a1c9ae0644f0c0394ad17c3ee36e6c0e42 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 28 Nov 2025 17:18:59 +0100 Subject: [PATCH 22/27] kola: refactor iso.pxe* tests --- mantle/kola/tests/iso/common.go | 227 +++------- mantle/kola/tests/iso/live-as-disk.go | 4 +- mantle/kola/tests/iso/live-fips.go | 4 +- mantle/kola/tests/iso/live-iscsi.go | 4 +- mantle/kola/tests/iso/live-iso.go | 53 +-- mantle/kola/tests/iso/live-login.go | 4 +- mantle/kola/tests/iso/live-pxe.go | 430 ++++++++++++++++--- mantle/platform/machine/qemu/metal.go | 571 -------------------------- 8 files changed, 456 insertions(+), 841 deletions(-) delete mode 100644 mantle/platform/machine/qemu/metal.go diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 73c518f6ff..13c816637c 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -6,24 +6,34 @@ import ( "io" "os" "path/filepath" - "strconv" "strings" "time" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" - "github.com/coreos/coreos-assembler/mantle/platform" - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" ) const ( installTimeoutMins = 12 + + // defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option + defaultQemuHostIPv4 = "10.0.2.2" ) +// This object gets serialized to YAML and fed to coreos-installer: +// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format +type CoreosInstallerConfig struct { + ImageURL string `yaml:"image-url,omitempty"` + IgnitionFile string `yaml:"ignition-file,omitempty"` + Insecure bool `yaml:"insecure,omitempty"` + AppendKargs []string `yaml:"append-karg,omitempty"` + CopyNetwork bool `yaml:"copy-network,omitempty"` + DestDevice string `yaml:"dest-device,omitempty"` + Console []string `yaml:"console,omitempty"` +} + type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") instInsecure bool @@ -62,114 +72,7 @@ func isoTest(name string, run func(c cluster.TestCluster), arch []string) *regis } } -func newBaseQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, error) { - builder := qemu.NewMetalQemuBuilderDefault() - if opts.enableUefiSecure { - builder.Firmware = "uefi-secure" - } else if opts.enableUefi { - builder.Firmware = "uefi" - } - - if err := os.MkdirAll(outdir, 0755); err != nil { - return nil, err - } - - builder.InheritConsole = opts.console - if !opts.console { - builder.ConsoleFile = filepath.Join(outdir, "console.txt") - } - - if kola.QEMUOptions.Memory != "" { - parsedMem, err := strconv.ParseInt(kola.QEMUOptions.Memory, 10, 32) - if err != nil { - return nil, err - } - builder.MemoryMiB = int(parsedMem) - } - - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if opts.pxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - return builder, nil -} - -func newQemuBuilder(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, err := newBaseQemuBuilder(opts, outdir) - if err != nil { - return nil, nil, err - } - - config, err := conf.EmptyIgnition().Render(conf.FailWarnings) - if err != nil { - return nil, nil, err - } - - err = forwardJournal(outdir, builder, config) - if err != nil { - return nil, nil, err - } - - return builder, config, nil -} - -func forwardJournal(outdir string, builder *platform.QemuBuilder, config *conf.Conf) error { - journalPipe, err := builder.VirtioJournal(config, "") - if err != nil { - return err - } - journalOut, err := os.OpenFile(filepath.Join(outdir, "journal.txt"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - go func() { - _, err := io.Copy(journalOut, journalPipe) - if err != nil && err != io.EOF { - fmt.Printf("error copying journal: %v\n", err) - } - }() - - return nil -} - -func newQemuBuilderWithDisk(opts IsoTestOpts, outdir string) (*platform.QemuBuilder, *conf.Conf, error) { - builder, config, err := newQemuBuilder(opts, outdir) - - if err != nil { - return nil, nil, err - } - - sectorSize := 0 - if opts.enable4k { - sectorSize = 4096 - } - - disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - MultiPathDisk: opts.enableMultipath, - } - - //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup - if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { - // s390x and aarch64 need to use bootindex as they don't support boot once - if err := builder.AddDisk(&disk); err != nil { - return nil, nil, err - } - } else { - if err := builder.AddPrimaryDisk(&disk); err != nil { - return nil, nil, err - } - } - - return builder, config, nil -} - -func CheckTestOutput(output *os.File, expected []string) error { +func checkTestOutput(output *os.File, expected []string) error { reader := bufio.NewReader(output) for _, exp := range expected { line, err := reader.ReadString('\n') @@ -192,8 +95,11 @@ func CheckTestOutput(output *os.File, expected []string) error { return nil } -func EnsureLiveArtifactsExist() error { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { +func ensureLiveArtifactsExist() error { + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { + return errors.Errorf("Build %s is missing live-iso artifacts\n", kola.CosaBuild.Meta.Name) + } + if kola.CosaBuild.Meta.BuildArtifacts.LiveRootfs == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs == nil { return errors.Errorf("Build %s is missing live artifacts\n", kola.CosaBuild.Meta.Name) } if kola.CosaBuild.Meta.BuildArtifacts.Metal == nil || kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative == nil { @@ -202,59 +108,54 @@ func EnsureLiveArtifactsExist() error { return nil } -func awaitCompletion(c cluster.TestCluster, inst *platform.QemuInstance, console bool, outdir string, qchan *os.File, booterrchan chan error, expected []string) error { - ctx := c.Context() +// Sometimes the logs that stream from various virtio streams can be +// incomplete because they depend on services inside the guest. +// When you are debugging earlyboot/initramfs issues this can be +// problematic. Let's add a hook here to enable more debugging. +func renderCosaTestIsoDebugKargs() []string { + if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { + return []string{"systemd.log_color=0", "systemd.log_level=debug", + "systemd.journald.forward_to_console=1", + "systemd.journald.max_level_console=debug"} + } else { + return []string{} + } +} + +func absSymlink(src, dest string) error { + src, err := filepath.Abs(src) + if err != nil { + return err + } + return os.Symlink(src, dest) +} - errchan := make(chan error) - go func() { - timeout := (time.Duration(installTimeoutMins*(100+kola.Options.ExtendTimeoutPercent)) * time.Minute) / 100 - time.Sleep(timeout) - errchan <- fmt.Errorf("timed out after %v", timeout) - }() - if !console { - go func() { - errBuf, err := inst.WaitIgnitionError(ctx) - if err == nil { - if errBuf != "" { - c.Logf("entered emergency.target in initramfs") - path := filepath.Join(outdir, "ignition-virtio-dump.txt") - if err := os.WriteFile(path, []byte(errBuf), 0644); err != nil { - c.Errorf("Failed to write journal: %v", err) - } - err = platform.ErrInitramfsEmergency - } - } - if err != nil { - errchan <- err - } - }() +// setupMetalImage creates a symlink to the metal image. +func setupMetalImage(builddir, metalimg, destdir string) (string, error) { + if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { + return "", err } - go func() { - err := inst.Wait() - // only one Wait() gets process data, so also manually check for signal - //plog.Debugf("qemu exited err=%v", err) - if err == nil && inst.Signaled() { - err = errors.New("process killed") - } + return metalimg, nil +} + +func cat(outfile string, infiles ...string) error { + out, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + defer out.Close() + for _, infile := range infiles { + in, err := os.Open(infile) if err != nil { - errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion") + return err } - time.Sleep(1 * time.Minute) - errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion") - }() - go func() { - errchan <- CheckTestOutput(qchan, expected) - }() - go func() { - //check for error when switching boot order - if booterrchan != nil { - if err := <-booterrchan; err != nil { - errchan <- err - } + defer in.Close() + _, err = io.Copy(out, in) + if err != nil { + return err } - }() - err := <-errchan - return err + } + return nil } var liveOKSignal = "live-test-OK" diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index 2895742b0e..a96dd51e6f 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -43,7 +43,7 @@ func isoAsDiskUefiSecure(c cluster.TestCluster) { } func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { - if err := EnsureLiveArtifactsExist(); err != nil { + if err := ensureLiveArtifactsExist(); err != nil { fmt.Println(err) return } @@ -84,7 +84,7 @@ func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { // Read line in a goroutine and send errors to channel go func() { - errchan <- CheckTestOutput(output, []string{liveOKSignal}) + errchan <- checkTestOutput(output, []string{liveOKSignal}) }() isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index 6a06f6d3e4..18becee2c9 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -40,7 +40,7 @@ ExecStart=grep FIPS etc/crypto-policies/config RequiredBy=fips-signal-ok.service` func testLiveFIPS(c cluster.TestCluster) { - if err := EnsureLiveArtifactsExist(); err != nil { + if err := ensureLiveArtifactsExist(); err != nil { fmt.Println(err) return } @@ -81,7 +81,7 @@ func testLiveFIPS(c cluster.TestCluster) { // Read line in a goroutine and send errors to channel go func() { - errchan <- CheckTestOutput(output, []string{liveOKSignal}) + errchan <- checkTestOutput(output, []string{liveOKSignal}) }() isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 3c5632cb86..97ac9251ea 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -89,7 +89,7 @@ var iscsi_butane_config string // - when the system is booted, write a success string to /dev/virtio-ports/testisocompletion // - as this serial device is mapped to the host serial device, the test concludes func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { - if err := EnsureLiveArtifactsExist(); err != nil { + if err := ensureLiveArtifactsExist(); err != nil { fmt.Println(err) return } @@ -173,7 +173,7 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { // Read line in a goroutine and send errors to channel go func() { - errchan <- CheckTestOutput(output, []string{"iscsi-boot-ok"}) + errchan <- checkTestOutput(output, []string{"iscsi-boot-ok"}) }() return nil diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 4ecc8ad857..3eea06728a 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -191,37 +191,8 @@ func isoOfflineInstallFromRam4kUefi(c cluster.TestCluster) { isoLiveInstall(c, opts) } -// Sometimes the logs that stream from various virtio streams can be -// incomplete because they depend on services inside the guest. -// When you are debugging earlyboot/initramfs issues this can be -// problematic. Let's add a hook here to enable more debugging. -func renderCosaTestIsoDebugKargs() []string { - if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { - return []string{"systemd.log_color=0", "systemd.log_level=debug", - "systemd.journald.forward_to_console=1", - "systemd.journald.max_level_console=debug"} - } else { - return []string{} - } -} - -// This object gets serialized to YAML and fed to coreos-installer: -// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format -type coreosInstallerConfig struct { - ImageURL string `yaml:"image-url,omitempty"` - IgnitionFile string `yaml:"ignition-file,omitempty"` - Insecure bool `yaml:"insecure,omitempty"` - AppendKargs []string `yaml:"append-karg,omitempty"` - CopyNetwork bool `yaml:"copy-network,omitempty"` - DestDevice string `yaml:"dest-device,omitempty"` - Console []string `yaml:"console,omitempty"` -} - -// defaultQemuHostIPv4 is documented in `man qemu-kvm`, under the `-netdev` option -const defaultQemuHostIPv4 = "10.0.2.2" - func isoLiveInstall(c cluster.TestCluster, opts IsoTestOpts) { - if err := EnsureLiveArtifactsExist(); err != nil { + if err := ensureLiveArtifactsExist(); err != nil { fmt.Println(err) return } @@ -278,7 +249,7 @@ func isoRunTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) - installerConfig := coreosInstallerConfig{ + installerConfig := CoreosInstallerConfig{ IgnitionFile: "/var/opt/pointer.ign", DestDevice: "/dev/vda", AppendKargs: renderCosaTestIsoDebugKargs(), @@ -472,12 +443,12 @@ func isoRunTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { errchan := make(chan error) go func() { - errchan <- CheckTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) + errchan <- checkTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) }() //check for error when switching boot order go func() { - if err := CheckTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { + if err := checkTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { errchan <- err return } @@ -491,22 +462,6 @@ func isoRunTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { return err } -func absSymlink(src, dest string) error { - src, err := filepath.Abs(src) - if err != nil { - return err - } - return os.Symlink(src, dest) -} - -// setupMetalImage creates a symlink to the metal image. -func setupMetalImage(builddir, metalimg, destdir string) (string, error) { - if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { - return "", err - } - return metalimg, nil -} - func createMiniso(tempd string, isopath string, url string) (string, error) { minisopath := filepath.Join(tempd, "minimal.iso") // This is obviously also available in the build dir, but to be realistic, diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index 0a81c2dbad..b7fdfa1869 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -21,7 +21,7 @@ func init() { } func testLiveLogin(c cluster.TestCluster, enableUefi bool, enableUefiSecure bool) { - if err := EnsureLiveArtifactsExist(); err != nil { + if err := ensureLiveArtifactsExist(); err != nil { fmt.Println(err) return } @@ -48,7 +48,7 @@ version: 1.1.0`) // Read line in a goroutine and send errors to channel go func() { - errchan <- CheckTestOutput(output, []string{"coreos-liveiso-success"}) + errchan <- checkTestOutput(output, []string{"coreos-liveiso-success"}) }() isopath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.LiveIso.Path) diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 95ac56a70d..8487090acf 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -2,15 +2,22 @@ package iso import ( "fmt" + "net" + "net/http" "os" + "os/exec" + "path/filepath" "strings" "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" - "github.com/coreos/coreos-assembler/mantle/util" + coreosarch "github.com/coreos/stream-metadata-go/arch" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) func init() { @@ -130,90 +137,413 @@ RequiredBy=coreos-installer.target ` func testPXE(c cluster.TestCluster, opts IsoTestOpts) { - if err := EnsureLiveArtifactsExist(); err != nil { + if err := ensureLiveArtifactsExist(); err != nil { fmt.Println(err) return } - - var outdir string - var qc *qemu.Cluster - - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - outdir = pc.RuntimeConf().OutputDir - qc = pc - default: - c.Fatalf("Unsupported cluster type") - } - if opts.addNmKeyfile { c.Fatal("--add-nm-keyfile not yet supported for PXE") } - inst := qemu.Install{ - CosaBuild: kola.CosaBuild, - NmKeyfiles: make(map[string]string), - Insecure: opts.instInsecure, - Native4k: opts.enable4k, - MultiPathDisk: opts.enableMultipath, - PxeAppendRootfs: opts.pxeAppendRootfs, + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + c.Fatalf("Unsupported cluster type") } - - tmpd, err := os.MkdirTemp("", "kola-iso.pxe") - if err != nil { - c.Fatal(err) + if opts.enable4k { + qc.EnforeNative4k() } - defer os.RemoveAll(tmpd) - sshPubKeyBuf, _, err := util.CreateSSHAuthorizedKey(tmpd) + installerConfig := CoreosInstallerConfig{ + Console: []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]}, + AppendKargs: renderCosaTestIsoDebugKargs(), + Insecure: opts.instInsecure, + } + keys, err := qc.Keys() if err != nil { c.Fatal(err) } - - builder, virtioJournalConfig, err := newQemuBuilderWithDisk(opts, outdir) + installerConfigData, err := yaml.Marshal(installerConfig) if err != nil { c.Fatal(err) } + mode := 0644 - // increase the memory for pxe tests with appended rootfs in the initrd - // we were bumping up into the 4GiB limit in RHCOS/c9s - // pxe-offline-install.rootfs-appended.bios tests - if inst.PxeAppendRootfs && builder.MemoryMiB < 5120 { - builder.MemoryMiB = 5120 - } - - inst.Builder = builder - completionChannel, err := inst.Builder.VirtioChannelRead("testisocompletion") + liveConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) if err != nil { - c.Fatal(err) // , "setting up virtio-serial channel") + c.Fatal(err) } - - var keys []string - keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf))) - virtioJournalConfig.AddAuthorizedKeys("core", keys) - - liveConfig := *virtioJournalConfig + liveConfig.CopyKeys(keys) liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - if opts.isOffline { contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.OstreeVersion) liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) } + // XXX: https://github.com/coreos/coreos-installer/issues/1171 + if coreosarch.CurrentRpmArch() != "s390x" { + liveConfig.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) + } + liveConfig.AddAutoLogin() + liveConfig.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) - targetConfig := *virtioJournalConfig + targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) + if err != nil { + c.Fatal(err) + } + targetConfig.CopyKeys(keys) targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - mach, err := inst.PXE(kola.QEMUOptions.PxeKernelArgs, liveConfig, targetConfig, opts.isOffline) + tempdir, err := os.MkdirTemp("/var/tmp", "mantle-pxe") if err != nil { c.Fatal(err) } - qc.AddMach(mach) + defer func() { + os.RemoveAll(tempdir) + }() - err = awaitCompletion(c, mach.Instance(), opts.console, outdir, completionChannel, mach.BootStartedErrorChannel(), []string{liveOKSignal, signalCompleteString}) + pxe, err := createPXE(tempdir, opts) + if err != nil { + c.Fatal(errors.Wrapf(err, "setting up install")) + } + + overrideFW := func(builder *platform.QemuBuilder) error { + if opts.enableUefi { + builder.Firmware = "uefi" + } + // increase the memory for pxe tests with appended rootfs in the initrd + // we were bumping up into the 4GiB limit in RHCOS/c9s + builder.MemoryMiB = 4096 + if opts.pxeAppendRootfs { + builder.MemoryMiB = 5120 + } + + if err := absSymlink(builder.ConfigFile, filepath.Join(pxe.tftpdir, "pxe-live.ign")); err != nil { + return err + } + + targetpath := filepath.Join(filepath.Dir(builder.ConfigFile), "pxe-target.ign") + if err := targetConfig.WriteFile(targetpath); err != nil { + return err + } + if err := absSymlink(targetpath, filepath.Join(pxe.tftpdir, "pxe-target.ign")); err != nil { + return err + } + // don't attach config to VM + builder.ConfigFile = "" + return nil + } + + setupNet := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", pxe.networkdevice) + if pxe.bootindex == "" { + builder.Append("-boot", "once=n") + } else { + netdev += fmt.Sprintf(",bootindex=%s", pxe.bootindex) + } + builder.Append("-device", netdev) + usernetdev := fmt.Sprintf("user,id=mynet0,tftp=%s,bootfile=%s", pxe.tftpdir, pxe.bootfile) + if pxe.tftpipaddr != "10.0.2.2" { + usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9" + } + builder.Append("-netdev", usernetdev) + return nil + } + + var isoCompletionOutput *os.File + var bootStartedOutput *os.File + setupDisks := func(_ platform.QemuMachineOptions, builder *platform.QemuBuilder) error { + sectorSize := 0 + if opts.enable4k { + sectorSize = 4096 + } + disk := platform.Disk{ + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: opts.enableMultipath, + } + //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup + if coreosarch.CurrentRpmArch() == "s390x" || coreosarch.CurrentRpmArch() == "aarch64" { + // s390x and aarch64 need to use bootindex as they don't support boot once + if err := builder.AddDisk(&disk); err != nil { + return err + } + } else { + if err := builder.AddPrimaryDisk(&disk); err != nil { + return err + } + } + isoCompletionOutput, err = builder.VirtioChannelRead("testisocompletion") + if err != nil { + return errors.Wrap(err, "setting up testisocompletion virtio-serial channel") + } + bootStartedOutput, err = builder.VirtioChannelRead("bootstarted") + if err != nil { + return errors.Wrap(err, "setting up bootstarted virtio-serial channel") + } + return nil + } + + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true + callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} + qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callacks) + if err != nil { + c.Fatal(errors.Wrap(err, "unable to create test machine")) + } + + errchan := make(chan error) + go func() { + errchan <- checkTestOutput(isoCompletionOutput, []string{liveOKSignal, signalCompleteString}) + }() + + //check for error when switching boot order + go func() { + if err := checkTestOutput(bootStartedOutput, []string{bootStartedSignal}); err != nil { + errchan <- err + return + } + if err := qc.Instance(qm).SwitchBootOrder(); err != nil { + errchan <- errors.Wrapf(err, "switching boot order failed") + return + } + }() + + err = <-errchan if err != nil { c.Fatal(err) } } + +type PXE struct { + tftpdir string + tftpipaddr string + boottype string + networkdevice string + bootindex string + pxeimagepath string + bootfile string +} + +func createPXE(tempdir string, opts IsoTestOpts) (*PXE, error) { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + rootfs := kola.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path + builddir := kola.CosaBuild.Dir + + tftpdir := filepath.Join(tempdir, "tftp") + if err := os.Mkdir(tftpdir, 0777); err != nil { + return nil, err + } + + for _, name := range []string{kernel, initramfs, rootfs} { + if err := absSymlink(filepath.Join(builddir, name), filepath.Join(tftpdir, name)); err != nil { + return nil, err + } + } + + if opts.pxeAppendRootfs { + // replace the initramfs symlink with a concatenation of + // the initramfs and rootfs + initrd := filepath.Join(tftpdir, initramfs) + if err := os.Remove(initrd); err != nil { + return nil, err + } + if err := cat(initrd, filepath.Join(builddir, initramfs), filepath.Join(builddir, rootfs)); err != nil { + return nil, err + } + } + + var metalimg string + if opts.enable4k { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path + } else { + metalimg = kola.CosaBuild.Meta.BuildArtifacts.Metal.Path + } + metalname, err := setupMetalImage(builddir, metalimg, tftpdir) + if err != nil { + return nil, errors.Wrapf(err, "setting up metal image") + } + + pxe := &PXE{ + tftpdir: tftpdir, + } + if err := pxe.setupArchDefaults(opts); err != nil { + return nil, err + } + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return nil, err + } + port := listener.Addr().(*net.TCPAddr).Port + baseurl := fmt.Sprintf("http://%s:%d", pxe.tftpipaddr, port) + + kargs := renderCosaTestIsoDebugKargs() + kargs = append(kargs, renderBaseKargs()...) + kargs = append(kargs, kola.QEMUOptions.PxeKernelArgs...) + kargs = append(kargs, fmt.Sprintf("ignition.config.url=%s/pxe-live.ign", baseurl)) + kargs = append(kargs, renderInstallKargs(baseurl, metalname, opts)...) + if rootfs != "" && !opts.pxeAppendRootfs { + kargs = append(kargs, fmt.Sprintf("coreos.live.rootfs_url=%s/%s", baseurl, rootfs)) + } + kargsStr := strings.Join(kargs, " ") + + switch pxe.boottype { + case "pxe": + if err := pxe.configBootPxe(kargsStr); err != nil { + return nil, err + } + case "grub": + if err := pxe.configBootGrub(kargsStr); err != nil { + return nil, err + } + default: + return nil, errors.Errorf("Unhandled boottype %s", pxe.boottype) + } + + //nolint // Yeah this leaks + go func() { + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tftpdir))) + http.Serve(listener, mux) + }() + + return pxe, nil +} + +func (pxe *PXE) setupArchDefaults(opts IsoTestOpts) error { + pxe.tftpipaddr = "192.168.76.2" + switch coreosarch.CurrentRpmArch() { + case "x86_64": + pxe.networkdevice = "e1000" + if opts.enableUefi { + pxe.boottype = "grub" + pxe.bootfile = "/boot/grub2/grubx64.efi" + pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubx64.efi" + // Choose bootindex=2. First boot the hard drive won't + // have an OS and will fall through to bootindex 2 (net) + pxe.bootindex = "2" + } else { + pxe.boottype = "pxe" + pxe.pxeimagepath = "/usr/share/syslinux/" + } + case "aarch64": + pxe.boottype = "grub" + pxe.networkdevice = "virtio-net-pci" + pxe.bootfile = "/boot/grub2/grubaa64.efi" + pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubaa64.efi" + pxe.bootindex = "1" + case "ppc64le": + pxe.boottype = "grub" + pxe.networkdevice = "virtio-net-pci" + pxe.bootfile = "/boot/grub2/powerpc-ieee1275/core.elf" + case "s390x": + pxe.boottype = "pxe" + pxe.networkdevice = "virtio-net-ccw" + pxe.bootindex = "1" + pxe.tftpipaddr = "10.0.2.2" + default: + return fmt.Errorf("unsupported arch %s", coreosarch.CurrentRpmArch()) + } + return nil +} + +func (pxe *PXE) configBootPxe(kargs string) error { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + + pxeconfigdir := filepath.Join(pxe.tftpdir, "pxelinux.cfg") + if err := os.Mkdir(pxeconfigdir, 0777); err != nil { + return errors.Wrapf(err, "creating dir %s", pxeconfigdir) + } + pxeimages := []string{"pxelinux.0", "ldlinux.c32"} + pxeconfig := []byte(fmt.Sprintf(` +DEFAULT pxeboot +TIMEOUT 20 +PROMPT 0 +LABEL pxeboot + KERNEL %s + APPEND initrd=%s %s +`, kernel, initramfs, kargs)) + if coreosarch.CurrentRpmArch() == "s390x" { + pxeconfig = []byte(kargs) + } + pxeconfig_path := filepath.Join(pxeconfigdir, "default") + if err := os.WriteFile(pxeconfig_path, pxeconfig, 0777); err != nil { + return errors.Wrapf(err, "writing file %s", pxeconfig_path) + } + + // this is only for s390x where the pxe image has to be created; + // s390 doesn't seem to have a pre-created pxe image although have to check on this + if pxe.pxeimagepath == "" { + kernelpath := filepath.Join(pxe.tftpdir, kernel) + initrdpath := filepath.Join(pxe.tftpdir, initramfs) + err := exec.Command("/usr/bin/mk-s390image", kernelpath, "-r", initrdpath, + "-p", filepath.Join(pxeconfigdir, "default"), filepath.Join(pxe.tftpdir, pxeimages[0])).Run() + if err != nil { + return errors.Wrap(err, "running mk-s390image") + } + } else { + for _, img := range pxeimages { + srcpath := filepath.Join("/usr/share/syslinux", img) + cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", srcpath, pxe.tftpdir) + cp_cmd.Stderr = os.Stderr + if err := cp_cmd.Run(); err != nil { + return errors.Wrapf(err, "running cp-reflink %s %s", srcpath, pxe.tftpdir) + } + } + } + pxe.bootfile = "/" + pxeimages[0] + return nil +} + +func (pxe *PXE) configBootGrub(kargs string) error { + kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path + initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path + + grub2_mknetdir_cmd := exec.Command("grub2-mknetdir", "--net-directory="+pxe.tftpdir) + grub2_mknetdir_cmd.Stderr = os.Stderr + if err := grub2_mknetdir_cmd.Run(); err != nil { + return errors.Wrap(err, "running grub2-mknetdir") + } + if pxe.pxeimagepath != "" { + dstpath := filepath.Join(pxe.tftpdir, "boot/grub2") + cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", pxe.pxeimagepath, dstpath) + cp_cmd.Stderr = os.Stderr + if err := cp_cmd.Run(); err != nil { + return errors.Wrapf(err, "running cp-reflink %s %s", pxe.pxeimagepath, dstpath) + } + } + if err := os.WriteFile(filepath.Join(pxe.tftpdir, "boot/grub2/grub.cfg"), []byte(fmt.Sprintf(` +default=0 +timeout=1 +menuentry "CoreOS (BIOS/UEFI)" { + echo "Loading kernel" + linux /%s %s + echo "Loading initrd" + initrd %s +}`, kernel, kargs, initramfs)), 0777); err != nil { + return errors.Wrap(err, "writing grub.cfg") + } + return nil +} + +func renderBaseKargs() []string { + baseKargs := []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} + return append(baseKargs, fmt.Sprintf("console=%s", platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()])) +} + +func renderInstallKargs(baseurl string, metalname string, opts IsoTestOpts) []string { + args := []string{"coreos.inst.install_dev=/dev/vda", + fmt.Sprintf("coreos.inst.ignition_url=%s/pxe-target.ign", baseurl)} + if !opts.isOffline { + args = append(args, fmt.Sprintf("coreos.inst.image_url=%s/%s", baseurl, metalname)) + } + // FIXME - ship signatures by default too + if opts.instInsecure { + args = append(args, "coreos.inst.insecure") + } + return args +} diff --git a/mantle/platform/machine/qemu/metal.go b/mantle/platform/machine/qemu/metal.go deleted file mode 100644 index f5d95ed345..0000000000 --- a/mantle/platform/machine/qemu/metal.go +++ /dev/null @@ -1,571 +0,0 @@ -// Copyright 2020 Red Hat -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package qemu - -import ( - "bufio" - "fmt" - "io" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/coreos/coreos-assembler/mantle/platform" - coreosarch "github.com/coreos/stream-metadata-go/arch" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" - - "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/system/exec" - "github.com/coreos/coreos-assembler/mantle/util" -) - -const bootStartedSignal = "boot-started-OK" - -// TODO derive this from docs, or perhaps include kargs in cosa metadata? -var baseKargs = []string{"rd.neednet=1", "ip=dhcp", "ignition.firstboot", "ignition.platform.id=metal"} - -var ( - bootStartedUnit = fmt.Sprintf(`[Unit] - Description=TestISO Boot Started - Requires=dev-virtio\\x2dports-bootstarted.device - OnFailure=emergency.target - OnFailureJobMode=isolate - [Service] - Type=oneshot - RemainAfterExit=yes - ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/bootstarted' - [Install] - RequiredBy=coreos-installer.target - `, bootStartedSignal) -) - -// NewMetalQemuBuilderDefault returns a QEMU builder instance with some -// defaults set up for bare metal. -func NewMetalQemuBuilderDefault() *platform.QemuBuilder { - builder := platform.NewQemuBuilder() - // https://github.com/coreos/fedora-coreos-tracker/issues/388 - // https://github.com/coreos/fedora-coreos-docs/pull/46 - builder.MemoryMiB = 4096 - return builder -} - -type Install struct { - CosaBuild *util.LocalBuild - Builder *platform.QemuBuilder - Insecure bool - Native4k bool - MultiPathDisk bool - PxeAppendRootfs bool - NmKeyfiles map[string]string - - // These are set by the install path - kargs []string - ignition conf.Conf - liveIgnition conf.Conf -} - -// Check that artifact has been built and locally exists -func (inst *Install) checkArtifactsExist(artifacts []string) error { - version := inst.CosaBuild.Meta.OstreeVersion - for _, name := range artifacts { - artifact, err := inst.CosaBuild.Meta.GetArtifact(name) - if err != nil { - return fmt.Errorf("Missing artifact %s for %s build: %s", name, version, err) - } - path := filepath.Join(inst.CosaBuild.Dir, artifact.Path) - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("Missing local file for artifact %s for build %s", name, version) - } - } - } - return nil -} - -func (inst *Install) PXE(kargs []string, liveIgnition, ignition conf.Conf, offline bool) (*machine, error) { - artifacts := []string{"live-kernel", "live-rootfs"} - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - - installerConfig := installerConfig{ - Console: []string{platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()]}, - AppendKargs: renderCosaTestIsoDebugKargs(), - } - installerConfigData, err := yaml.Marshal(installerConfig) - if err != nil { - return nil, err - } - mode := 0644 - - // XXX: https://github.com/coreos/coreos-installer/issues/1171 - if coreosarch.CurrentRpmArch() != "s390x" { - liveIgnition.AddFile("/etc/coreos/installer.d/mantle.yaml", string(installerConfigData), mode) - } - - inst.kargs = append(renderCosaTestIsoDebugKargs(), kargs...) - inst.ignition = ignition - inst.liveIgnition = liveIgnition - - mach, err := inst.runPXE(&kernelSetup{ - kernel: inst.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path, - initramfs: inst.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path, - rootfs: inst.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path, - }, offline) - if err != nil { - return nil, errors.Wrapf(err, "testing live installer") - } - - return mach, nil -} - -type kernelSetup struct { - kernel, initramfs, rootfs string -} - -type pxeSetup struct { - tftpipaddr string - boottype string - networkdevice string - bootindex string - pxeimagepath string - - // bootfile is initialized later - bootfile string -} - -type installerRun struct { - inst *Install - builder *platform.QemuBuilder - - builddir string - tempdir string - tftpdir string - - metalimg string - metalname string - - baseurl string - - kern kernelSetup - pxe pxeSetup -} - -func absSymlink(src, dest string) error { - src, err := filepath.Abs(src) - if err != nil { - return err - } - return os.Symlink(src, dest) -} - -// setupMetalImage creates a symlink to the metal image. -func setupMetalImage(builddir, metalimg, destdir string) (string, error) { - if err := absSymlink(filepath.Join(builddir, metalimg), filepath.Join(destdir, metalimg)); err != nil { - return "", err - } - return metalimg, nil -} - -func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) { - var artifacts []string - if inst.Native4k { - artifacts = append(artifacts, "metal4k") - } else { - artifacts = append(artifacts, "metal") - } - if err := inst.checkArtifactsExist(artifacts); err != nil { - return nil, err - } - - builder := inst.Builder - - tempdir, err := os.MkdirTemp("/var/tmp", "mantle-pxe") - if err != nil { - return nil, err - } - cleanupTempdir := true - defer func() { - if cleanupTempdir { - os.RemoveAll(tempdir) - } - }() - - tftpdir := filepath.Join(tempdir, "tftp") - if err := os.Mkdir(tftpdir, 0777); err != nil { - return nil, err - } - - builddir := inst.CosaBuild.Dir - if err := inst.ignition.WriteFile(filepath.Join(tftpdir, "config.ign")); err != nil { - return nil, err - } - // This code will ensure to add an SSH key to `pxe-live.ign` config. - inst.liveIgnition.AddAutoLogin() - inst.liveIgnition.AddSystemdUnit("boot-started.service", bootStartedUnit, conf.Enable) - if err := inst.liveIgnition.WriteFile(filepath.Join(tftpdir, "pxe-live.ign")); err != nil { - return nil, err - } - - for _, name := range []string{kern.kernel, kern.initramfs, kern.rootfs} { - if err := absSymlink(filepath.Join(builddir, name), filepath.Join(tftpdir, name)); err != nil { - return nil, err - } - } - if inst.PxeAppendRootfs { - // replace the initramfs symlink with a concatenation of - // the initramfs and rootfs - initrd := filepath.Join(tftpdir, kern.initramfs) - if err := os.Remove(initrd); err != nil { - return nil, err - } - if err := cat(initrd, filepath.Join(builddir, kern.initramfs), filepath.Join(builddir, kern.rootfs)); err != nil { - return nil, err - } - } - - var metalimg string - if inst.Native4k { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path - } else { - metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal.Path - } - metalname, err := setupMetalImage(builddir, metalimg, tftpdir) - if err != nil { - return nil, errors.Wrapf(err, "setting up metal image") - } - - pxe := pxeSetup{} - pxe.tftpipaddr = "192.168.76.2" - switch coreosarch.CurrentRpmArch() { - case "x86_64": - pxe.networkdevice = "e1000" - if builder.Firmware == "uefi" { - pxe.boottype = "grub" - pxe.bootfile = "/boot/grub2/grubx64.efi" - pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubx64.efi" - // Choose bootindex=2. First boot the hard drive won't - // have an OS and will fall through to bootindex 2 (net) - pxe.bootindex = "2" - } else { - pxe.boottype = "pxe" - pxe.pxeimagepath = "/usr/share/syslinux/" - } - case "aarch64": - pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" - pxe.bootfile = "/boot/grub2/grubaa64.efi" - pxe.pxeimagepath = "/boot/efi/EFI/fedora/grubaa64.efi" - pxe.bootindex = "1" - case "ppc64le": - pxe.boottype = "grub" - pxe.networkdevice = "virtio-net-pci" - pxe.bootfile = "/boot/grub2/powerpc-ieee1275/core.elf" - case "s390x": - pxe.boottype = "pxe" - pxe.networkdevice = "virtio-net-ccw" - pxe.tftpipaddr = "10.0.2.2" - pxe.bootindex = "1" - default: - return nil, fmt.Errorf("Unsupported arch %s", coreosarch.CurrentRpmArch()) - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tftpdir))) - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - port := listener.Addr().(*net.TCPAddr).Port - //nolint // Yeah this leaks - go func() { - http.Serve(listener, mux) - }() - baseurl := fmt.Sprintf("http://%s:%d", pxe.tftpipaddr, port) - - cleanupTempdir = false // Transfer ownership - return &installerRun{ - inst: inst, - - builder: builder, - tempdir: tempdir, - tftpdir: tftpdir, - builddir: builddir, - - metalimg: metalimg, - metalname: metalname, - - baseurl: baseurl, - - pxe: pxe, - kern: *kern, - }, nil -} - -func renderBaseKargs() []string { - return append(baseKargs, fmt.Sprintf("console=%s", platform.ConsoleKernelArgument[coreosarch.CurrentRpmArch()])) -} - -func renderInstallKargs(t *installerRun, offline bool) []string { - args := []string{"coreos.inst.install_dev=/dev/vda", - fmt.Sprintf("coreos.inst.ignition_url=%s/config.ign", t.baseurl)} - if !offline { - args = append(args, fmt.Sprintf("coreos.inst.image_url=%s/%s", t.baseurl, t.metalname)) - } - // FIXME - ship signatures by default too - if t.inst.Insecure { - args = append(args, "coreos.inst.insecure") - } - return args -} - -// Sometimes the logs that stream from various virtio streams can be -// incomplete because they depend on services inside the guest. -// When you are debugging earlyboot/initramfs issues this can be -// problematic. Let's add a hook here to enable more debugging. -func renderCosaTestIsoDebugKargs() []string { - if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok { - return []string{"systemd.log_color=0", "systemd.log_level=debug", - "systemd.journald.forward_to_console=1", - "systemd.journald.max_level_console=debug"} - } else { - return []string{} - } -} - -func (t *installerRun) destroy() error { - t.builder.Close() - if t.tempdir != "" { - return os.RemoveAll(t.tempdir) - } - return nil -} - -func (t *installerRun) completePxeSetup(kargs []string) error { - if t.kern.rootfs != "" && !t.inst.PxeAppendRootfs { - kargs = append(kargs, fmt.Sprintf("coreos.live.rootfs_url=%s/%s", t.baseurl, t.kern.rootfs)) - } - kargsStr := strings.Join(kargs, " ") - - switch t.pxe.boottype { - case "pxe": - pxeconfigdir := filepath.Join(t.tftpdir, "pxelinux.cfg") - if err := os.Mkdir(pxeconfigdir, 0777); err != nil { - return errors.Wrapf(err, "creating dir %s", pxeconfigdir) - } - pxeimages := []string{"pxelinux.0", "ldlinux.c32"} - pxeconfig := []byte(fmt.Sprintf(` - DEFAULT pxeboot - TIMEOUT 20 - PROMPT 0 - LABEL pxeboot - KERNEL %s - APPEND initrd=%s %s - `, t.kern.kernel, t.kern.initramfs, kargsStr)) - if coreosarch.CurrentRpmArch() == "s390x" { - pxeconfig = []byte(kargsStr) - } - pxeconfig_path := filepath.Join(pxeconfigdir, "default") - if err := os.WriteFile(pxeconfig_path, pxeconfig, 0777); err != nil { - return errors.Wrapf(err, "writing file %s", pxeconfig_path) - } - - // this is only for s390x where the pxe image has to be created; - // s390 doesn't seem to have a pre-created pxe image although have to check on this - if t.pxe.pxeimagepath == "" { - kernelpath := filepath.Join(t.tftpdir, t.kern.kernel) - initrdpath := filepath.Join(t.tftpdir, t.kern.initramfs) - err := exec.Command("/usr/bin/mk-s390image", kernelpath, "-r", initrdpath, - "-p", filepath.Join(pxeconfigdir, "default"), filepath.Join(t.tftpdir, pxeimages[0])).Run() - if err != nil { - return errors.Wrap(err, "running mk-s390image") - } - } else { - for _, img := range pxeimages { - srcpath := filepath.Join("/usr/share/syslinux", img) - cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", srcpath, t.tftpdir) - cp_cmd.Stderr = os.Stderr - if err := cp_cmd.Run(); err != nil { - return errors.Wrapf(err, "running cp-reflink %s %s", srcpath, t.tftpdir) - } - } - } - t.pxe.bootfile = "/" + pxeimages[0] - case "grub": - grub2_mknetdir_cmd := exec.Command("grub2-mknetdir", "--net-directory="+t.tftpdir) - grub2_mknetdir_cmd.Stderr = os.Stderr - if err := grub2_mknetdir_cmd.Run(); err != nil { - return errors.Wrap(err, "running grub2-mknetdir") - } - if t.pxe.pxeimagepath != "" { - dstpath := filepath.Join(t.tftpdir, "boot/grub2") - cp_cmd := exec.Command("/usr/lib/coreos-assembler/cp-reflink", t.pxe.pxeimagepath, dstpath) - cp_cmd.Stderr = os.Stderr - if err := cp_cmd.Run(); err != nil { - return errors.Wrapf(err, "running cp-reflink %s %s", t.pxe.pxeimagepath, dstpath) - } - } - if err := os.WriteFile(filepath.Join(t.tftpdir, "boot/grub2/grub.cfg"), []byte(fmt.Sprintf(` - default=0 - timeout=1 - menuentry "CoreOS (BIOS/UEFI)" { - echo "Loading kernel" - linux /%s %s - echo "Loading initrd" - initrd %s - } - `, t.kern.kernel, kargsStr, t.kern.initramfs)), 0777); err != nil { - return errors.Wrap(err, "writing grub.cfg") - } - default: - panic("Unhandled boottype " + t.pxe.boottype) - } - - return nil -} - -func switchBootOrderSignal(qinst *platform.QemuInstance, bootstartedchan *os.File, booterrchan *chan error) { - *booterrchan = make(chan error) - go func() { - err := qinst.Wait() - // only one Wait() gets process data, so also manually check for signal - if err == nil && qinst.Signaled() { - err = errors.New("process killed") - } - if err != nil { - *booterrchan <- errors.Wrapf(err, "QEMU unexpectedly exited while waiting for %s", bootStartedSignal) - } - }() - go func() { - r := bufio.NewReader(bootstartedchan) - l, err := r.ReadString('\n') - if err != nil { - if err == io.EOF { - // this may be from QEMU getting killed or exiting; wait a bit - // to give a chance for .Wait() above to feed the channel with a - // better error - time.Sleep(1 * time.Second) - *booterrchan <- fmt.Errorf("Got EOF from boot started channel, %s expected", bootStartedSignal) - } else { - *booterrchan <- errors.Wrapf(err, "reading from boot started channel") - } - return - } - line := strings.TrimSpace(l) - // switch the boot order here, we are well into the installation process - only for aarch64 and s390x - if line == bootStartedSignal { - if err := qinst.SwitchBootOrder(); err != nil { - *booterrchan <- errors.Wrapf(err, "switching boot order failed") - return - } - } - // OK! - *booterrchan <- nil - }() -} - -func cat(outfile string, infiles ...string) error { - out, err := os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - defer out.Close() - for _, infile := range infiles { - in, err := os.Open(infile) - if err != nil { - return err - } - defer in.Close() - _, err = io.Copy(out, in) - if err != nil { - return err - } - } - return nil -} - -func (t *installerRun) run() (*platform.QemuInstance, error) { - builder := t.builder - netdev := fmt.Sprintf("%s,netdev=mynet0,mac=52:54:00:12:34:56", t.pxe.networkdevice) - if t.pxe.bootindex == "" { - builder.Append("-boot", "once=n") - } else { - netdev += fmt.Sprintf(",bootindex=%s", t.pxe.bootindex) - } - builder.Append("-device", netdev) - usernetdev := fmt.Sprintf("user,id=mynet0,tftp=%s,bootfile=%s", t.tftpdir, t.pxe.bootfile) - if t.pxe.tftpipaddr != "10.0.2.2" { - usernetdev += ",net=192.168.76.0/24,dhcpstart=192.168.76.9" - } - builder.Append("-netdev", usernetdev) - - inst, err := builder.Exec() - if err != nil { - return nil, err - } - return inst, nil -} - -func (inst *Install) runPXE(kern *kernelSetup, offline bool) (*machine, error) { - t, err := inst.setup(kern) - if err != nil { - return nil, errors.Wrapf(err, "setting up install") - } - defer func() { - err = t.destroy() - }() - - bootStartedChan, err := inst.Builder.VirtioChannelRead("bootstarted") - if err != nil { - return nil, errors.Wrapf(err, "setting up bootstarted virtio-serial channel") - } - - kargs := renderBaseKargs() - kargs = append(kargs, inst.kargs...) - kargs = append(kargs, fmt.Sprintf("ignition.config.url=%s/pxe-live.ign", t.baseurl)) - - kargs = append(kargs, renderInstallKargs(t, offline)...) - if err := t.completePxeSetup(kargs); err != nil { - return nil, errors.Wrapf(err, "completing PXE setup") - } - qinst, err := t.run() - if err != nil { - return nil, errors.Wrapf(err, "running PXE install") - } - tempdir := t.tempdir - t.tempdir = "" // Transfer ownership - instmachine := machine{ - inst: qinst, - tempdir: tempdir, - } - switchBootOrderSignal(qinst, bootStartedChan, &instmachine.bootStartedErrorChannel) - return &instmachine, nil -} - -// This object gets serialized to YAML and fed to coreos-installer: -// https://coreos.github.io/coreos-installer/customizing-install/#config-file-format -type installerConfig struct { - ImageURL string `yaml:"image-url,omitempty"` - IgnitionFile string `yaml:"ignition-file,omitempty"` - Insecure bool `yaml:",omitempty"` - AppendKargs []string `yaml:"append-karg,omitempty"` - CopyNetwork bool `yaml:"copy-network,omitempty"` - DestDevice string `yaml:"dest-device,omitempty"` - Console []string `yaml:"console,omitempty"` -} From 370503b39e1bd870d9dd73db4e3dc2f885f8426f Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Fri, 28 Nov 2025 17:48:58 +0100 Subject: [PATCH 23/27] machine: drop no longer used code --- mantle/platform/machine/qemu/machine.go | 37 +++++-------------------- 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/mantle/platform/machine/qemu/machine.go b/mantle/platform/machine/qemu/machine.go index bb428838af..92488314fc 100644 --- a/mantle/platform/machine/qemu/machine.go +++ b/mantle/platform/machine/qemu/machine.go @@ -26,29 +26,19 @@ import ( ) type machine struct { - qc *Cluster - id string - inst *platform.QemuInstance - journal *platform.Journal - consolePath string - console string - ip string - tempdir string - bootStartedErrorChannel chan error + qc *Cluster + id string + inst *platform.QemuInstance + journal *platform.Journal + consolePath string + console string + ip string } func (m *machine) ID() string { return m.id } -func (m *machine) Instance() *platform.QemuInstance { - return m.inst -} - -func (m *machine) BootStartedErrorChannel() chan error { - return m.bootStartedErrorChannel -} - func (m *machine) IP() string { return m.ip } @@ -101,15 +91,6 @@ func (m *machine) WaitForSoftReboot(timeout time.Duration, oldSoftRebootsCount s return platform.WaitForMachineSoftReboot(m, m.journal, timeout, oldSoftRebootsCount) } -func (m *machine) DeleteTempdir() error { - var err error = nil - if m.tempdir != "" { - err = os.RemoveAll(m.tempdir) - m.tempdir = "" - } - return err -} - func (m *machine) Destroy() { if m.inst != nil { m.inst.Destroy() @@ -132,10 +113,6 @@ func (m *machine) Destroy() { if m.qc != nil { m.qc.DelMach(m) } - - if err := m.DeleteTempdir(); err != nil { - plog.Errorf("Error removing tempdir for instance %v: %v", m.ID(), err) - } } func (m *machine) ConsoleOutput() string { From 2e033a85543fe4fb36da2827d98850889dc89882 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 1 Dec 2025 09:26:08 +0100 Subject: [PATCH 24/27] kola: iso.* fix typos --- mantle/kola/tests/iso/live-as-disk.go | 4 ++-- mantle/kola/tests/iso/live-fips.go | 4 ++-- mantle/kola/tests/iso/live-iscsi.go | 4 ++-- mantle/kola/tests/iso/live-iso.go | 4 ++-- mantle/kola/tests/iso/live-login.go | 4 ++-- mantle/kola/tests/iso/live-pxe.go | 6 +++--- mantle/platform/machine/qemu/cluster.go | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index a96dd51e6f..89fa10c2d1 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -91,8 +91,8 @@ func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { return builder.AddIso(isopath, "", true) } - callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} - _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callacks) + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callbacks) if err != nil { c.Fatalf("Unable to create test machine: %v", err) } diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index 18becee2c9..0d16e6d4dc 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -88,8 +88,8 @@ func testLiveFIPS(c cluster.TestCluster) { return builder.AddIso(isopath, "", false) } - callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} - _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callacks) + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callbacks) if err != nil { c.Fatalf("Unable to create test machine: %v", err) } diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 97ac9251ea..42c0e7807b 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -179,8 +179,8 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { return nil } - callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} - _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callacks) + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callbacks) if err != nil { c.Fatalf("Unable to create test machine: %v", err) } diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 3eea06728a..daa73604ab 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -208,10 +208,10 @@ func isoLiveInstall(c cluster.TestCluster, opts IsoTestOpts) { c.Fatalf("Unsupported cluster type") } if opts.enable4k { - qc.EnforeNative4k() + qc.EnforceNative4k() } if opts.enableMultipath { - qc.EnforeMultipath() + qc.EnforceMultipath() } tempdir, err := os.MkdirTemp("/var/tmp", "iso") diff --git a/mantle/kola/tests/iso/live-login.go b/mantle/kola/tests/iso/live-login.go index b7fdfa1869..6338773906 100644 --- a/mantle/kola/tests/iso/live-login.go +++ b/mantle/kola/tests/iso/live-login.go @@ -57,8 +57,8 @@ version: 1.1.0`) switch pc := c.Cluster.(type) { case *qemu.Cluster: - callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} - _, err := pc.NewMachineWithQemuOptionsAndBuilderCallbacks(butane, platform.QemuMachineOptions{}, callacks) + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} + _, err := pc.NewMachineWithQemuOptionsAndBuilderCallbacks(butane, platform.QemuMachineOptions{}, callbacks) if err != nil { c.Fatalf("Unable to create test machine: %v", err) } diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 8487090acf..827150fd33 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -150,7 +150,7 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { c.Fatalf("Unsupported cluster type") } if opts.enable4k { - qc.EnforeNative4k() + qc.EnforceNative4k() } installerConfig := CoreosInstallerConfig{ @@ -287,8 +287,8 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { extra := platform.QemuMachineOptions{} extra.SkipStartMachine = true - callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} - qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callacks) + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} + qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callbacks) if err != nil { c.Fatal(errors.Wrap(err, "unable to create test machine")) } diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index 590667b2a8..cab3e1ddc9 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -51,11 +51,11 @@ type BuilderCallbacks struct { OverrideDefaults func(builder *platform.QemuBuilder) error } -func (qc *Cluster) EnforeNative4k() { +func (qc *Cluster) EnforceNative4k() { qc.flight.opts.Native4k = true } -func (qc *Cluster) EnforeMultipath() { +func (qc *Cluster) EnforceMultipath() { qc.flight.opts.MultiPathDisk = true } From 3c7e00cee2b7ffc48d6fce3709407718f1c25290 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 1 Dec 2025 09:39:20 +0100 Subject: [PATCH 25/27] kola: iso.*: don't leak http server --- mantle/kola/tests/iso/live-iso.go | 45 +++++++++++++++++++------------ mantle/kola/tests/iso/live-pxe.go | 23 +++++++++++----- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index daa73604ab..3beb87c2fa 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -1,8 +1,10 @@ package iso import ( + "context" _ "embed" "fmt" + "log" "net" "net/http" "os" @@ -203,17 +205,6 @@ func isoLiveInstall(c cluster.TestCluster, opts IsoTestOpts) { c.Fatal("Cannot use `--add-nm-keyfile` with offline mode") } - qc, ok := c.Cluster.(*qemu.Cluster) - if !ok { - c.Fatalf("Unsupported cluster type") - } - if opts.enable4k { - qc.EnforceNative4k() - } - if opts.enableMultipath { - qc.EnforceMultipath() - } - tempdir, err := os.MkdirTemp("/var/tmp", "iso") if err != nil { c.Fatal(err) @@ -222,12 +213,22 @@ func isoLiveInstall(c cluster.TestCluster, opts IsoTestOpts) { os.RemoveAll(tempdir) }() - if err := isoRunTest(qc, opts, tempdir); err != nil { + if err := isoRunTest(c, opts, tempdir); err != nil { c.Fatal(err) } } -func isoRunTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { +func isoRunTest(c cluster.TestCluster, opts IsoTestOpts, tempdir string) error { + qc, ok := c.Cluster.(*qemu.Cluster) + if !ok { + return errors.Errorf("Unsupported cluster type") + } + if opts.enable4k { + qc.EnforceNative4k() + } + if opts.enableMultipath { + qc.EnforceMultipath() + } keys, err := qc.Keys() if err != nil { return err @@ -320,11 +321,21 @@ func isoRunTest(qc *qemu.Cluster, opts IsoTestOpts, tempdir string) error { targetConfig.AddConfigSource(baseurl + "/target.ign") serializedTargetConfig = targetConfig.String() - //nolint // Yeah this leaks + ctx := c.Context() + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tempdir))) + srv := &http.Server{Handler: mux} + + go func() { + if err := srv.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("http serve: %v", err) + } + }() + + // stop server when ctx is canceled go func() { - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tempdir))) - http.Serve(listener, mux) + <-ctx.Done() + _ = srv.Shutdown(context.Background()) }() } diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index 827150fd33..aa763928b9 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -1,7 +1,9 @@ package iso import ( + "context" "fmt" + "log" "net" "net/http" "os" @@ -203,7 +205,7 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { os.RemoveAll(tempdir) }() - pxe, err := createPXE(tempdir, opts) + pxe, err := createPXE(c.Context(), tempdir, opts) if err != nil { c.Fatal(errors.Wrapf(err, "setting up install")) } @@ -326,7 +328,7 @@ type PXE struct { bootfile string } -func createPXE(tempdir string, opts IsoTestOpts) (*PXE, error) { +func createPXE(ctx context.Context, tempdir string, opts IsoTestOpts) (*PXE, error) { kernel := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path initramfs := kola.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path rootfs := kola.CosaBuild.Meta.BuildArtifacts.LiveRootfs.Path @@ -403,11 +405,20 @@ func createPXE(tempdir string, opts IsoTestOpts) (*PXE, error) { return nil, errors.Errorf("Unhandled boottype %s", pxe.boottype) } - //nolint // Yeah this leaks + mux := http.NewServeMux() + mux.Handle("/", http.FileServer(http.Dir(tftpdir))) + srv := &http.Server{Handler: mux} + + go func() { + if err := srv.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("http serve: %v", err) + } + }() + + // stop server when ctx is canceled go func() { - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(tftpdir))) - http.Serve(listener, mux) + <-ctx.Done() + _ = srv.Shutdown(context.Background()) }() return pxe, nil From 89dc604c850ef8faa33f47e3defc5607bd75b20f Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Mon, 1 Dec 2025 09:42:00 +0100 Subject: [PATCH 26/27] kola: drop unused 'console' option from iso.* tests --- mantle/kola/tests/iso/common.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mantle/kola/tests/iso/common.go b/mantle/kola/tests/iso/common.go index 13c816637c..09123e0c35 100644 --- a/mantle/kola/tests/iso/common.go +++ b/mantle/kola/tests/iso/common.go @@ -36,9 +36,7 @@ type CoreosInstallerConfig struct { type IsoTestOpts struct { // Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - instInsecure bool - // Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - console bool + instInsecure bool addNmKeyfile bool enable4k bool enableMultipath bool From ac38a76925b7ac2da3194c7c25906d070cd33510 Mon Sep 17 00:00:00 2001 From: Nikita Dubrovskii Date: Tue, 2 Dec 2025 14:23:53 +0100 Subject: [PATCH 27/27] kola: set SkipStartMachine and drop unused SSH keys for iso.* tests The iso.* tests use systemd units that report success or failure via virtio channels and always power off the machine. Because of this, we do not need to SSH into the instance or check Ignition errors. --- mantle/kola/tests/iso/live-as-disk.go | 9 +++------ mantle/kola/tests/iso/live-fips.go | 9 +++------ mantle/kola/tests/iso/live-iscsi.go | 10 ++++------ mantle/kola/tests/iso/live-iso.go | 11 +++-------- mantle/kola/tests/iso/live-pxe.go | 7 +------ 5 files changed, 14 insertions(+), 32 deletions(-) diff --git a/mantle/kola/tests/iso/live-as-disk.go b/mantle/kola/tests/iso/live-as-disk.go index 89fa10c2d1..cc8a838c28 100644 --- a/mantle/kola/tests/iso/live-as-disk.go +++ b/mantle/kola/tests/iso/live-as-disk.go @@ -59,11 +59,6 @@ func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { } config.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) config.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) - keys, err := qc.Keys() - if err != nil { - c.Fatal(err) - } - config.CopyKeys(keys) overrideFW := func(builder *platform.QemuBuilder) error { switch { @@ -91,8 +86,10 @@ func isoTestAsDisk(c cluster.TestCluster, opts IsoTestOpts) { return builder.AddIso(isopath, "", true) } + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} - _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callbacks) + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, extra, callbacks) if err != nil { c.Fatalf("Unable to create test machine: %v", err) } diff --git a/mantle/kola/tests/iso/live-fips.go b/mantle/kola/tests/iso/live-fips.go index 0d16e6d4dc..4a848fee83 100644 --- a/mantle/kola/tests/iso/live-fips.go +++ b/mantle/kola/tests/iso/live-fips.go @@ -57,11 +57,6 @@ func testLiveFIPS(c cluster.TestCluster) { config.AddSystemdUnit("fips-verify.service", fipsVerify, conf.Enable) config.AddSystemdUnit("fips-signal-ok.service", liveSignalOKUnit, conf.Enable) config.AddSystemdUnit("fips-emergency-target.service", signalFailureUnit, conf.Enable) - keys, err := qc.Keys() - if err != nil { - c.Fatal(err) - } - config.CopyKeys(keys) overrideFW := func(builder *platform.QemuBuilder) error { builder.Firmware = "uefi" @@ -88,8 +83,10 @@ func testLiveFIPS(c cluster.TestCluster) { return builder.AddIso(isopath, "", false) } + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} - _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callbacks) + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, extra, callbacks) if err != nil { c.Fatalf("Unable to create test machine: %v", err) } diff --git a/mantle/kola/tests/iso/live-iscsi.go b/mantle/kola/tests/iso/live-iscsi.go index 42c0e7807b..9c504c8a6d 100644 --- a/mantle/kola/tests/iso/live-iscsi.go +++ b/mantle/kola/tests/iso/live-iscsi.go @@ -113,11 +113,7 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { if err != nil { c.Fatal(err) } - keys, err := qc.Keys() - if err != nil { - c.Fatal(err) - } - config.CopyKeys(keys) + // Add a failure target to stop the test if something go wrong rather than waiting for the 10min timeout config.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) config.MountHost("/var/cosaroot", true) @@ -179,8 +175,10 @@ func isoInstalliScsi(c cluster.TestCluster, opts IsoTestOpts) { return nil } + extra := platform.QemuMachineOptions{} + extra.SkipStartMachine = true callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, OverrideDefaults: overrideFW} - _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, platform.QemuMachineOptions{}, callbacks) + _, err = qc.NewMachineWithQemuOptionsAndBuilderCallbacks(config, extra, callbacks) if err != nil { c.Fatalf("Unable to create test machine: %v", err) } diff --git a/mantle/kola/tests/iso/live-iso.go b/mantle/kola/tests/iso/live-iso.go index 3beb87c2fa..dfdd5fd334 100644 --- a/mantle/kola/tests/iso/live-iso.go +++ b/mantle/kola/tests/iso/live-iso.go @@ -229,15 +229,11 @@ func isoRunTest(c cluster.TestCluster, opts IsoTestOpts, tempdir string) error { if opts.enableMultipath { qc.EnforceMultipath() } - keys, err := qc.Keys() - if err != nil { - return err - } + targetConfig, err := conf.EmptyIgnition().Render(conf.FailWarnings) if err != nil { return err } - targetConfig.CopyKeys(keys) targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) @@ -359,7 +355,6 @@ func isoRunTest(c cluster.TestCluster, opts IsoTestOpts, tempdir string) error { if err != nil { return err } - liveConfig.CopyKeys(keys) liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable) liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable) @@ -446,8 +441,8 @@ func isoRunTest(c cluster.TestCluster, opts IsoTestOpts, tempdir string) error { extra := platform.QemuMachineOptions{} extra.SkipStartMachine = true - callacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} - qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callacks) + callbacks := qemu.BuilderCallbacks{SetupDisks: setupDisks, SetupNetwork: setupNet, OverrideDefaults: overrideFW} + qm, err := qc.NewMachineWithQemuOptionsAndBuilderCallbacks(liveConfig, extra, callbacks) if err != nil { return errors.Wrap(err, "unable to create test machine") } diff --git a/mantle/kola/tests/iso/live-pxe.go b/mantle/kola/tests/iso/live-pxe.go index aa763928b9..6b5ffca4f3 100644 --- a/mantle/kola/tests/iso/live-pxe.go +++ b/mantle/kola/tests/iso/live-pxe.go @@ -160,10 +160,7 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { AppendKargs: renderCosaTestIsoDebugKargs(), Insecure: opts.instInsecure, } - keys, err := qc.Keys() - if err != nil { - c.Fatal(err) - } + installerConfigData, err := yaml.Marshal(installerConfig) if err != nil { c.Fatal(err) @@ -174,7 +171,6 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { if err != nil { c.Fatal(err) } - liveConfig.CopyKeys(keys) liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) if opts.isOffline { @@ -192,7 +188,6 @@ func testPXE(c cluster.TestCluster, opts IsoTestOpts) { if err != nil { c.Fatal(err) } - targetConfig.CopyKeys(keys) targetConfig.AddSystemdUnit("coreos-test-installer.service", signalCompletionUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable)