vm-test-run-nix-grpc-farm
checks.aarch64-linux.farm
· build #487
· raw
1tribuchet: building on eliza2Machine state will be reset. To keep it, pass --keep-machine-state3start all VLans4(finished: start all VLans, in 0.00 seconds)5Test will time out and terminate in 900.0 seconds6run the VM test script7start all VMs8client: starting vm9lb: starting vm10worker1: starting vm11worker2: starting vm12worker2 # Disk image does not exist, creating the virtualisation disk image...13worker2 # Formatting '/build/vm-state-worker2/tmp.KBZ5ABMFaw', fmt=raw size=107374182414worker2 # mke2fs 1.47.4 (6-Mar-2025)15worker2 # Discarding device blocks: 0/262144 done16worker2: QEMU running (pid 52)17client # Disk image does not exist, creating the virtualisation disk image...18client: QEMU running (pid 48)19worker1 # Disk image does not exist, creating the virtualisation disk image...20lb: QEMU running (pid 50)21worker1: QEMU running (pid 51)22worker2 # Creating filesystem with 262144 4k blocks and 65536 inodes23client # Formatting '/build/vm-state-client/tmp.2OVDiyig1j', fmt=raw size=107374182424worker1 # Formatting '/build/vm-state-worker1/tmp.Z7jC4h1bMB', fmt=raw size=107374182425lb # Disk image does not exist, creating the virtualisation disk image...26worker2 # Filesystem UUID: 1d096151-ab1f-41e9-a555-24244c6f0ffb27worker1 # mke2fs 1.47.4 (6-Mar-2025)28lb # Formatting '/build/vm-state-lb/tmp.DuKWkGDLho', fmt=raw size=107374182429worker2 # Superblock backups stored on blocks:30worker1 # Discarding device blocks: 0/262144 done31lb # mke2fs 1.47.4 (6-Mar-2025)32worker2 # 32768, 98304, 163840, 22937633worker1 # Creating filesystem with 262144 4k blocks and 65536 inodes34lb # Discarding device blocks: 0/262144 done35worker2 # 36worker1 # Filesystem UUID: 09dc7d4e-8077-4e0f-bde3-c958c06e6d2137lb # Creating filesystem with 262144 4k blocks and 65536 inodes38worker2 # Allocating group tables: 0/8 done39worker1 # Superblock backups stored on blocks:40lb # Filesystem UUID: 41855766-2557-42a8-8a8c-02099e3fa24741worker2 # Writing inode tables: 0/8 done42worker1 # 32768, 98304, 163840, 22937643lb # Superblock backups stored on blocks:44worker2 # Creating journal (8192 blocks): done45worker1 # 46lb # 32768, 98304, 163840, 22937647worker2 # Writing superblocks and filesystem accounting information: 0/8 done48worker1 # Allocating group tables: 0/8 done49lb # 50worker2 # 51worker1 # Writing inode tables: 0/8 done52lb # Allocating group tables: 0/8 done53worker2 # Virtualisation disk image created.54worker1 # Creating journal (8192 blocks): done55lb # Writing inode tables: 0/8 done56(finished: start all VMs, in 0.49 seconds)57client # mke2fs 1.47.4 (6-Mar-2025)58worker1 # Writing superblocks and filesystem accounting information: 0/8 done59lb # Creating journal (8192 blocks): done60client: waiting for unit network-addresses-eth1.service61client # Discarding device blocks: 0/262144 done62worker1 # 63lb # Writing superblocks and filesystem accounting information: 0/8 done64client: waiting for the VM to finish booting65client # Creating filesystem with 262144 4k blocks and 65536 inodes66worker1 # Virtualisation disk image created.67client # Filesystem UUID: a3fd6980-f152-46a5-9741-a73ba2c0de4d68lb # 69client # Superblock backups stored on blocks:70lb # Virtualisation disk image created.71client # 32768, 98304, 163840, 22937672client # 73client # Allocating group tables: 0/8 done74client # Writing inode tables: 0/8 done75client # Creating journal (8192 blocks): done76client # Writing superblocks and filesystem accounting information: 0/8 done77client # 78client # Virtualisation disk image created.79worker2 # [ 0.000000] Booting Linux on physical CPU 0x0000000000 [0xc00fac40]80worker2 # [ 0.000000] Linux version 6.18.50 (nixbld@localhost) (gcc (GCC) 15.3.0, GNU ld (GNU Binutils) 2.46) #1-NixOS SMP PREEMPT Mon Sep 7 15:23:00 UTC 202681worker2 # [ 0.000000] KASLR enabled82worker2 # [ 0.000000] random: crng init done83worker2 # [ 0.000000] Machine model: linux,dummy-virt84worker2 # [ 0.000000] efi: UEFI not found.85worker2 # [ 0.000000] OF: reserved mem: Reserved memory: No reserved-memory node in the DT86worker2 # [ 0.000000] NUMA: Faking a node at [mem 0x0000000040000000-0x000000009fffffff]87worker2 # [ 0.000000] NODE_DATA(0) allocated [mem 0x9fdec700-0x9fdefe7f]88worker2 # [ 0.000000] Zone ranges:89worker2 # [ 0.000000] DMA [mem 0x0000000040000000-0x000000009fffffff]90worker2 # [ 0.000000] DMA32 empty91worker2 # [ 0.000000] Normal empty92worker2 # [ 0.000000] Device empty93worker2 # [ 0.000000] Movable zone start for each node94worker2 # [ 0.000000] Early memory node ranges95worker2 # [ 0.000000] node 0: [mem 0x0000000040000000-0x000000009fffffff]96worker2 # [ 0.000000] Initmem setup node 0 [mem 0x0000000040000000-0x000000009fffffff]97worker2 # [ 0.000000] cma: Reserved 32 MiB at 0x000000009c40000098worker2 # [ 0.000000] psci: probing for conduit method from DT.99worker2 # [ 0.000000] psci: PSCIv1.3 detected in firmware.100worker2 # [ 0.000000] psci: Using standard PSCI v0.2 function IDs101worker2 # [ 0.000000] psci: Trusted OS migration not required102worker2 # [ 0.000000] psci: SMC Calling Convention v1.1103worker2 # [ 0.000000] smccc: KVM: hypervisor services detected (0x00000000 0x00000000 0x00000000 0x00000003)104lb # [ 0.000000] Booting Linux on physical CPU 0x0000000000 [0xc00fac40]105worker2 # [ 0.000000] percpu: Embedded 76 pages/cpu s186648 r8192 d116456 u311296106worker2 # [ 0.000000] Detected PIPT I-cache on CPU0107lb # [ 0.000000] Linux version 6.18.50 (nixbld@localhost) (gcc (GCC) 15.3.0, GNU ld (GNU Binutils) 2.46) #1-NixOS SMP PREEMPT Mon Sep 7 15:23:00 UTC 2026108worker2 # [ 0.000000] CPU features: detected: Address authentication (architected QARMA5 algorithm)109lb # [ 0.000000] KASLR enabled110lb # [ 0.000000] random: crng init done111worker2 # [ 0.000000] CPU features: detected: GICv3 CPU interface112lb # [ 0.000000] Machine model: linux,dummy-virt113worker2 # [ 0.000000] CPU features: detected: Spectre-v4114lb # [ 0.000000] efi: UEFI not found.115worker2 # [ 0.000000] CPU features: detected: Spectre-BHB116lb # [ 0.000000] OF: reserved mem: Reserved memory: No reserved-memory node in the DT117worker2 # [ 0.000000] CPU features: detected: AmpereOne erratum AC03_CPU_38118lb # [ 0.000000] NUMA: Faking a node at [mem 0x0000000040000000-0x000000009fffffff]119worker2 # [ 0.000000] CPU features: detected: AmpereOne erratum AC04_CPU_23120lb # [ 0.000000] NODE_DATA(0) allocated [mem 0x9fdec700-0x9fdefe7f]121worker2 # [ 0.000000] alternatives: applying boot alternatives122lb # [ 0.000000] Zone ranges:123lb # [ 0.000000] DMA [mem 0x0000000040000000-0x000000009fffffff]124lb # [ 0.000000] DMA32 empty125lb # [ 0.000000] Normal empty126lb # [ 0.000000] Device empty127lb # [ 0.000000] Movable zone start for each node128lb # [ 0.000000] Early memory node ranges129worker2 # [ 0.000000] Kernel command line: console=ttyAMA0 console=tty0 panic=1 boot.panic_on_fail clocksource=acpi_pm root=fstab loglevel=7 net.ifnames=0 lsm=landlock,yama,bpf init=/nix/store/jp4jh0qijhgzz195bcnphlrraq8lrjyd-nixos-system-worker2-test/init regInfo=/nix/store/jccz8gpa5ww1qfn1xc1zwd1hn9yjjiyg-closure-info/registration console=ttyAMA0,115200n8 console=tty0130lb # [ 0.000000] node 0: [mem 0x0000000040000000-0x000000009fffffff]131lb # [ 0.000000] Initmem setup node 0 [mem 0x0000000040000000-0x000000009fffffff]132lb # [ 0.000000] cma: Reserved 32 MiB at 0x000000009c400000133worker2 # [ 0.000000] Unknown kernel command line parameters "regInfo=/nix/store/jccz8gpa5ww1qfn1xc1zwd1hn9yjjiyg-closure-info/registration", will be passed to user space.134lb # [ 0.000000] psci: probing for conduit method from DT.135worker2 # [ 0.000000] printk: log buffer data + meta data: 131072 + 458752 = 589824 bytes136lb # [ 0.000000] psci: PSCIv1.3 detected in firmware.137lb # [ 0.000000] psci: Using standard PSCI v0.2 function IDs138worker2 # [ 0.000000] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes, linear)139lb # [ 0.000000] psci: Trusted OS migration not required140lb # [ 0.000000] psci: SMC Calling Convention v1.1141worker2 # [ 0.000000] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes, linear)142worker2 # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size adjusted to 1MB143lb # [ 0.000000] smccc: KVM: hypervisor services detected (0x00000000 0x00000000 0x00000000 0x00000003)144worker1 # [ 0.000000] Booting Linux on physical CPU 0x0000000000 [0xc00fac40]145worker2 # [ 0.000000] software IO TLB: area num 1.146lb # [ 0.000000] percpu: Embedded 76 pages/cpu s186648 r8192 d116456 u311296147worker2 # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size roundup to 2MB148lb # [ 0.000000] Detected PIPT I-cache on CPU0149worker1 # [ 0.000000] Linux version 6.18.50 (nixbld@localhost) (gcc (GCC) 15.3.0, GNU ld (GNU Binutils) 2.46) #1-NixOS SMP PREEMPT Mon Sep 7 15:23:00 UTC 2026150worker1 # [ 0.000000] KASLR enabled151worker2 # [ 0.000000] software IO TLB: mapped [mem 0x000000009c000000-0x000000009c200000] (2MB)152lb # [ 0.000000] CPU features: detected: Address authentication (architected QARMA5 algorithm)153worker1 # [ 0.000000] random: crng init done154worker2 # [ 0.000000] Fallback order for Node 0: 0155worker1 # [ 0.000000] Machine model: linux,dummy-virt156lb # [ 0.000000] CPU features: detected: GICv3 CPU interface157worker1 # [ 0.000000] efi: UEFI not found.158worker2 # [ 0.000000] Built 1 zonelists, mobility grouping on. Total pages: 393216159lb # [ 0.000000] CPU features: detected: Spectre-v4160worker2 # [ 0.000000] Policy zone: DMA161lb # [ 0.000000] CPU features: detected: Spectre-BHB162worker1 # [ 0.000000] OF: reserved mem: Reserved memory: No reserved-memory node in the DT163worker2 # [ 0.000000] mem auto-init: stack:all(zero), heap alloc:on, heap free:off164lb # [ 0.000000] CPU features: detected: AmpereOne erratum AC03_CPU_38165worker1 # [ 0.000000] NUMA: Faking a node at [mem 0x0000000040000000-0x000000009fffffff]166worker2 # [ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1167lb # [ 0.000000] CPU features: detected: AmpereOne erratum AC04_CPU_23168worker1 # [ 0.000000] NODE_DATA(0) allocated [mem 0x9fdec700-0x9fdefe7f]169worker2 # [ 0.000000] allocated 3145728 bytes of page_ext170worker1 # [ 0.000000] Zone ranges:171lb # [ 0.000000] alternatives: applying boot alternatives172worker2 # [ 0.000000] ftrace: allocating 74885 entries in 294 pages173worker1 # [ 0.000000] DMA [mem 0x0000000040000000-0x000000009fffffff]174worker1 # [ 0.000000] DMA32 empty175worker2 # [ 0.000000] ftrace: allocated 294 pages with 4 groups176client # [ 0.000000] Booting Linux on physical CPU 0x0000000000 [0xc00fac40]177worker1 # [ 0.000000] Normal empty178worker2 # [ 0.000000] rcu: Hierarchical RCU implementation.179worker1 # [ 0.000000] Device empty180worker2 # [ 0.000000] rcu: RCU event tracing is enabled.181worker1 # [ 0.000000] Movable zone start for each node182client # [ 0.000000] Linux version 6.18.50 (nixbld@localhost) (gcc (GCC) 15.3.0, GNU ld (GNU Binutils) 2.46) #1-NixOS SMP PREEMPT Mon Sep 7 15:23:00 UTC 2026183worker1 # [ 0.000000] Early memory node ranges184client # [ 0.000000] KASLR enabled185worker2 # [ 0.000000] rcu: RCU restricting CPUs from NR_CPUS=384 to nr_cpu_ids=1.186client # [ 0.000000] random: crng init done187worker1 # [ 0.000000] node 0: [mem 0x0000000040000000-0x000000009fffffff]188worker2 # [ 0.000000] Trampoline variant of Tasks RCU enabled.189lb # [ 0.000000] Kernel command line: console=ttyAMA0 console=tty0 panic=1 boot.panic_on_fail clocksource=acpi_pm root=fstab loglevel=7 net.ifnames=0 lsm=landlock,yama,bpf init=/nix/store/mmnw0b9xdzgvlxb4mzaa4gjdldj887b1-nixos-system-lb-test/init regInfo=/nix/store/mmqsdvj4977m1gdgnvij2v3i1inliqqm-closure-info/registration console=ttyAMA0,115200n8 console=tty0190client # [ 0.000000] Machine model: linux,dummy-virt191worker2 # [ 0.000000] Rude variant of Tasks RCU enabled.192client # [ 0.000000] efi: UEFI not found.193worker1 # [ 0.000000] Initmem setup node 0 [mem 0x0000000040000000-0x000000009fffffff]194worker2 # [ 0.000000] Tracing variant of Tasks RCU enabled.195worker1 # [ 0.000000] cma: Reserved 32 MiB at 0x000000009c400000196client # [ 0.000000] OF: reserved mem: Reserved memory: No reserved-memory node in the DT197lb # [ 0.000000] Unknown kernel command line parameters "regInfo=/nix/store/mmqsdvj4977m1gdgnvij2v3i1inliqqm-closure-info/registration", will be passed to user space.198worker1 # [ 0.000000] psci: probing for conduit method from DT.199worker2 # [ 0.000000] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.200client # [ 0.000000] NUMA: Faking a node at [mem 0x0000000040000000-0x000000009fffffff]201worker1 # [ 0.000000] psci: PSCIv1.3 detected in firmware.202worker2 # [ 0.000000] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1203client # [ 0.000000] NODE_DATA(0) allocated [mem 0x9fdec700-0x9fdefe7f]204worker1 # [ 0.000000] psci: Using standard PSCI v0.2 function IDs205lb # [ 0.000000] printk: log buffer data + meta data: 131072 + 458752 = 589824 bytes206client # [ 0.000000] Zone ranges:207worker1 # [ 0.000000] psci: Trusted OS migration not required208worker2 # [ 0.000000] RCU Tasks: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.209client # [ 0.000000] DMA [mem 0x0000000040000000-0x000000009fffffff]210worker1 # [ 0.000000] psci: SMC Calling Convention v1.1211lb # [ 0.000000] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes, linear)212client # [ 0.000000] DMA32 empty213client # [ 0.000000] Normal empty214worker2 # [ 0.000000] RCU Tasks Rude: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.215lb # [ 0.000000] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes, linear)216client # [ 0.000000] Device empty217worker1 # [ 0.000000] smccc: KVM: hypervisor services detected (0x00000000 0x00000000 0x00000000 0x00000003)218client # [ 0.000000] Movable zone start for each node219lb # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size adjusted to 1MB220worker1 # [ 0.000000] percpu: Embedded 76 pages/cpu s186648 r8192 d116456 u311296221worker2 # [ 0.000000] RCU Tasks Trace: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.222client # [ 0.000000] Early memory node ranges223lb # [ 0.000000] software IO TLB: area num 1.224worker1 # [ 0.000000] Detected PIPT I-cache on CPU0225worker2 # [ 0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0226client # [ 0.000000] node 0: [mem 0x0000000040000000-0x000000009fffffff]227worker2 # [ 0.000000] GICv3: 256 SPIs implemented228lb # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size roundup to 2MB229worker1 # [ 0.000000] CPU features: detected: Address authentication (architected QARMA5 algorithm)230worker2 # [ 0.000000] GICv3: 0 Extended SPIs implemented231client # [ 0.000000] Initmem setup node 0 [mem 0x0000000040000000-0x000000009fffffff]232lb # [ 0.000000] software IO TLB: mapped [mem 0x000000009c000000-0x000000009c200000] (2MB)233worker1 # [ 0.000000] CPU features: detected: GICv3 CPU interface234worker2 # [ 0.000000] Root IRQ handler: gic_handle_irq235client # [ 0.000000] cma: Reserved 32 MiB at 0x000000009c400000236lb # [ 0.000000] Fallback order for Node 0: 0237worker1 # [ 0.000000] CPU features: detected: Spectre-v4238worker2 # [ 0.000000] GICv3: GICv3 features: 16 PPIs, DirectLPI239client # [ 0.000000] psci: probing for conduit method from DT.240worker1 # [ 0.000000] CPU features: detected: Spectre-BHB241worker2 # [ 0.000000] GICv3: GICD_CTLR.DS=1, SCR_EL3.FIQ=0242lb # [ 0.000000] Built 1 zonelists, mobility grouping on. Total pages: 393216243client # [ 0.000000] psci: PSCIv1.3 detected in firmware.244lb # [ 0.000000] Policy zone: DMA245worker1 # [ 0.000000] CPU features: detected: AmpereOne erratum AC03_CPU_38246client # [ 0.000000] psci: Using standard PSCI v0.2 function IDs247worker2 # [ 0.000000] GICv3: CPU0: found redistributor 0 region 0:0x00000000080a0000248lb # [ 0.000000] mem auto-init: stack:all(zero), heap alloc:on, heap free:off249worker1 # [ 0.000000] CPU features: detected: AmpereOne erratum AC04_CPU_23250client # [ 0.000000] psci: Trusted OS migration not required251worker2 # [ 0.000000] ITS [mem 0x08080000-0x0809ffff]252client # [ 0.000000] psci: SMC Calling Convention v1.1253worker1 # [ 0.000000] alternatives: applying boot alternatives254lb # [ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1255worker2 # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Devices @449f0000 (indirect, esz 8, psz 64K, shr 1)256lb # [ 0.000000] allocated 3145728 bytes of page_ext257client # [ 0.000000] smccc: KVM: hypervisor services detected (0x00000000 0x00000000 0x00000000 0x00000003)258lb # [ 0.000000] ftrace: allocating 74885 entries in 294 pages259worker2 # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Interrupt Collections @44a00000 (flat, esz 8, psz 64K, shr 1)260lb # [ 0.000000] ftrace: allocated 294 pages with 4 groups261client # [ 0.000000] percpu: Embedded 76 pages/cpu s186648 r8192 d116456 u311296262client # [ 0.000000] Detected PIPT I-cache on CPU0263worker2 # [ 0.000000] GICv3: using LPI property table @0x0000000044a10000264lb # [ 0.000000] rcu: Hierarchical RCU implementation.265lb # [ 0.000000] rcu: RCU event tracing is enabled.266worker2 # [ 0.000000] GICv3: CPU0: using allocated LPI pending table @0x0000000044a20000267client # [ 0.000000] CPU features: detected: Address authentication (architected QARMA5 algorithm)268lb # [ 0.000000] rcu: RCU restricting CPUs from NR_CPUS=384 to nr_cpu_ids=1.269client # [ 0.000000] CPU features: detected: GICv3 CPU interface270worker1 # [ 0.000000] Kernel command line: console=ttyAMA0 console=tty0 panic=1 boot.panic_on_fail clocksource=acpi_pm root=fstab loglevel=7 net.ifnames=0 lsm=landlock,yama,bpf init=/nix/store/ziy0h189df8ya90jkq7zpm1pm8f2w4a2-nixos-system-worker1-test/init regInfo=/nix/store/y0s6v94dcjwi6kvwhmwia3h44r79n8d1-closure-info/registration console=ttyAMA0,115200n8 console=tty0271worker2 # [ 0.000000] rcu: srcu_init: Setting srcu_struct sizes based on contention.272lb # [ 0.000000] Trampoline variant of Tasks RCU enabled.273client # [ 0.000000] CPU features: detected: Spectre-v4274lb # [ 0.000000] Rude variant of Tasks RCU enabled.275client # [ 0.000000] CPU features: detected: Spectre-BHB276worker2 # [ 0.000000] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns277lb # [ 0.000000] Tracing variant of Tasks RCU enabled.278worker1 # [ 0.000000] Unknown kernel command line parameters "regInfo=/nix/store/y0s6v94dcjwi6kvwhmwia3h44r79n8d1-closure-info/registration", will be passed to user space.279client # [ 0.000000] CPU features: detected: AmpereOne erratum AC03_CPU_38280worker2 # [ 0.000000] arch_timer: cp15 timer running at 1000.00MHz (virt).281lb # [ 0.000000] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.282client # [ 0.000000] CPU features: detected: AmpereOne erratum AC04_CPU_23283worker1 # [ 0.000000] printk: log buffer data + meta data: 131072 + 458752 = 589824 bytes284client # [ 0.000000] alternatives: applying boot alternatives285lb # [ 0.000000] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1286worker2 # [ 0.000000] clocksource: arch_sys_counter: mask: 0x1fffffffffffffff max_cycles: 0x1cd42e4dffb, max_idle_ns: 881590591483 ns287worker1 # [ 0.000000] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes, linear)288lb # [ 0.000000] RCU Tasks: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.289worker2 # [ 0.000000] sched_clock: 61 bits at 1000MHz, resolution 1ns, wraps every 4398046511103ns290worker1 # [ 0.000000] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes, linear)291worker2 # [ 0.000033] arm-pv: using stolen time PV292lb # [ 0.000000] RCU Tasks Rude: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.293worker1 # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size adjusted to 1MB294worker2 # [ 0.000645] kfence: initialized - using 2097152 bytes for 255 objects at 0x(____ptrval____)-0x(____ptrval____)295worker1 # [ 0.000000] software IO TLB: area num 1.296lb # [ 0.000000] RCU Tasks Trace: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.297worker2 # [ 0.000811] Console: colour dummy device 80x25298client # [ 0.000000] Kernel command line: console=ttyAMA0 console=tty0 panic=1 boot.panic_on_fail clocksource=acpi_pm root=fstab loglevel=7 net.ifnames=0 lsm=landlock,yama,bpf init=/nix/store/r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test/init regInfo=/nix/store/n9qcbxrz5wrg2b8xjvm71hhnbm3b8bvk-closure-info/registration console=ttyAMA0,115200n8 console=tty0299worker2 # [ 0.000819] printk: legacy console [tty0] enabled300worker1 # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size roundup to 2MB301lb # [ 0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0302lb # [ 0.000000] GICv3: 256 SPIs implemented303worker1 # [ 0.000000] software IO TLB: mapped [mem 0x000000009c000000-0x000000009c200000] (2MB)304lb # [ 0.000000] GICv3: 0 Extended SPIs implemented305worker2 # [ 0.001009] Calibrating delay loop (skipped), value calculated using timer frequency.. 2000.00 BogoMIPS (lpj=4000000)306worker1 # [ 0.000000] Fallback order for Node 0: 0307lb # [ 0.000000] Root IRQ handler: gic_handle_irq308client # [ 0.000000] Unknown kernel command line parameters "regInfo=/nix/store/n9qcbxrz5wrg2b8xjvm71hhnbm3b8bvk-closure-info/registration", will be passed to user space.309worker2 # [ 0.001016] pid_max: default: 32768 minimum: 301310lb # [ 0.000000] GICv3: GICv3 features: 16 PPIs, DirectLPI311worker1 # [ 0.000000] Built 1 zonelists, mobility grouping on. Total pages: 393216312client # [ 0.000000] printk: log buffer data + meta data: 131072 + 458752 = 589824 bytes313worker2 # [ 0.001109] LSM: initializing lsm=capability,landlock,yama,bpf,ima314worker1 # [ 0.000000] Policy zone: DMA315lb # [ 0.000000] GICv3: GICD_CTLR.DS=1, SCR_EL3.FIQ=0316worker2 # [ 0.001290] landlock: Up and running.317client # [ 0.000000] Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes, linear)318worker1 # [ 0.000000] mem auto-init: stack:all(zero), heap alloc:on, heap free:off319lb # [ 0.000000] GICv3: CPU0: found redistributor 0 region 0:0x00000000080a0000320worker2 # [ 0.001292] Yama: becoming mindful.321worker2 # [ 0.001941] LSM support for eBPF active322lb # [ 0.000000] ITS [mem 0x08080000-0x0809ffff]323worker1 # [ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1324client # [ 0.000000] Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes, linear)325worker2 # [ 0.002074] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)326worker1 # [ 0.000000] allocated 3145728 bytes of page_ext327lb # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Devices @449e0000 (indirect, esz 8, psz 64K, shr 1)328worker1 # [ 0.000000] ftrace: allocating 74885 entries in 294 pages329worker2 # [ 0.002107] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)330client # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size adjusted to 1MB331worker1 # [ 0.000000] ftrace: allocated 294 pages with 4 groups332lb # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Interrupt Collections @449f0000 (flat, esz 8, psz 64K, shr 1)333client # [ 0.000000] software IO TLB: area num 1.334worker2 # [ 0.003232] cacheinfo: Unable to detect cache hierarchy for CPU 0335worker1 # [ 0.000000] rcu: Hierarchical RCU implementation.336worker2 # [ 0.003956] rcu: Hierarchical SRCU implementation.337lb # [ 0.000000] GICv3: using LPI property table @0x0000000044a10000338worker1 # [ 0.000000] rcu: RCU event tracing is enabled.339client # [ 0.000000] software IO TLB: SWIOTLB bounce buffer size roundup to 2MB340worker2 # [ 0.003961] rcu: Max phase no-delay instances is 1000.341lb # [ 0.000000] GICv3: CPU0: using allocated LPI pending table @0x0000000044a20000342worker1 # [ 0.000000] rcu: RCU restricting CPUs from NR_CPUS=384 to nr_cpu_ids=1.343client # [ 0.000000] software IO TLB: mapped [mem 0x000000009c000000-0x000000009c200000] (2MB)344worker2 # [ 0.005159] fsl-mc MSI: its@8080000 domain created345client # [ 0.000000] Fallback order for Node 0: 0346worker1 # [ 0.000000] Trampoline variant of Tasks RCU enabled.347lb # [ 0.000000] rcu: srcu_init: Setting srcu_struct sizes based on contention.348worker2 # [ 0.005248] EFI services will not be available.349worker1 # [ 0.000000] Rude variant of Tasks RCU enabled.350worker2 # [ 0.005312] smp: Bringing up secondary CPUs ...351client # [ 0.000000] Built 1 zonelists, mobility grouping on. Total pages: 393216352worker1 # [ 0.000000] Tracing variant of Tasks RCU enabled.353client # [ 0.000000] Policy zone: DMA354worker2 # [ 0.005320] smp: Brought up 1 node, 1 CPU355lb # [ 0.000000] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns356worker2 # [ 0.005323] SMP: Total of 1 processors activated.357client # [ 0.000000] mem auto-init: stack:all(zero), heap alloc:on, heap free:off358lb # [ 0.000000] arch_timer: cp15 timer running at 1000.00MHz (virt).359worker1 # [ 0.000000] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.360worker2 # [ 0.005329] CPU: All CPU(s) started at EL1361client # [ 0.000000] SLUB: HWalign=64, Order=0-3, MinObjects=0, CPUs=1, Nodes=1362worker1 # [ 0.000000] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1363worker2 # [ 0.005340] CPU features: detected: Branch Target Identification364client # [ 0.000000] allocated 3145728 bytes of page_ext365lb # [ 0.000000] clocksource: arch_sys_counter: mask: 0x1fffffffffffffff max_cycles: 0x1cd42e4dffb, max_idle_ns: 881590591483 ns366worker2 # [ 0.005345] CPU features: detected: ARMv8.4 Translation Table Level367client # [ 0.000000] ftrace: allocating 74885 entries in 294 pages368worker1 # [ 0.000000] RCU Tasks: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.369lb # [ 0.000000] sched_clock: 61 bits at 1000MHz, resolution 1ns, wraps every 4398046511103ns370client # [ 0.000000] ftrace: allocated 294 pages with 4 groups371lb # [ 0.000033] arm-pv: using stolen time PV372worker2 # [ 0.005348] CPU features: detected: Instruction cache invalidation not required for I/D coherence373worker1 # [ 0.000000] RCU Tasks Rude: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.374client # [ 0.000000] rcu: Hierarchical RCU implementation.375client # [ 0.000000] rcu: RCU event tracing is enabled.376lb # [ 0.000572] kfence: initialized - using 2097152 bytes for 255 objects at 0x(____ptrval____)-0x(____ptrval____)377worker2 # [ 0.005352] CPU features: detected: Data cache clean to the PoU not required for I/D coherence378worker1 # [ 0.000000] RCU Tasks Trace: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.379client # [ 0.000000] rcu: RCU restricting CPUs from NR_CPUS=384 to nr_cpu_ids=1.380lb # [ 0.000724] Console: colour dummy device 80x25381worker2 # [ 0.005355] CPU features: detected: Common not Private translations382worker1 # [ 0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0383client # [ 0.000000] Trampoline variant of Tasks RCU enabled.384lb # [ 0.000733] printk: legacy console [tty0] enabled385worker1 # [ 0.000000] GICv3: 256 SPIs implemented386worker2 # [ 0.005359] CPU features: detected: CRC32 instructions387client # [ 0.000000] Rude variant of Tasks RCU enabled.388worker1 # [ 0.000000] GICv3: 0 Extended SPIs implemented389worker2 # [ 0.005361] CPU features: detected: Data cache clean to Point of Deep Persistence390client # [ 0.000000] Tracing variant of Tasks RCU enabled.391worker1 # [ 0.000000] Root IRQ handler: gic_handle_irq392lb # [ 0.000926] Calibrating delay loop (skipped), value calculated using timer frequency.. 2000.00 BogoMIPS (lpj=4000000)393worker1 # [ 0.000000] GICv3: GICv3 features: 16 PPIs, DirectLPI394lb # [ 0.000933] pid_max: default: 32768 minimum: 301395worker2 # [ 0.005365] CPU features: detected: Data cache clean to Point of Persistence396client # [ 0.000000] rcu: RCU calculated value of scheduler-enlistment delay is 25 jiffies.397worker1 # [ 0.000000] GICv3: GICD_CTLR.DS=1, SCR_EL3.FIQ=0398lb # [ 0.001001] LSM: initializing lsm=capability,landlock,yama,bpf,ima399worker2 # [ 0.005368] CPU features: detected: Data independent timing control (DIT)400client # [ 0.000000] rcu: Adjusting geometry for rcu_fanout_leaf=16, nr_cpu_ids=1401lb # [ 0.001203] landlock: Up and running.402worker1 # [ 0.000000] GICv3: CPU0: found redistributor 0 region 0:0x00000000080a0000403lb # [ 0.001206] Yama: becoming mindful.404worker1 # [ 0.000000] ITS [mem 0x08080000-0x0809ffff]405client # [ 0.000000] RCU Tasks: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.406lb # [ 0.001760] LSM support for eBPF active407worker2 # [ 0.005371] CPU features: detected: E0PD408lb # [ 0.001918] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)409worker1 # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Devices @449e0000 (indirect, esz 8, psz 64K, shr 1)410worker2 # [ 0.005374] CPU features: detected: Enhanced Counter Virtualization411client # [ 0.000000] RCU Tasks Rude: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.412lb # [ 0.001951] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)413worker2 # [ 0.005377] CPU features: detected: Enhanced Counter Virtualization (CNTPOFF)414client # [ 0.000000] RCU Tasks Trace: Setting shift to 0 and lim to 1 rcu_task_cb_adjust=1 rcu_task_cpu_ids=1.415worker1 # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Interrupt Collections @449f0000 (flat, esz 8, psz 64K, shr 1)416worker2 # [ 0.005380] CPU features: detected: Enhanced Virtualization Traps417lb # [ 0.002985] cacheinfo: Unable to detect cache hierarchy for CPU 0418client # [ 0.000000] NR_IRQS: 64, nr_irqs: 64, preallocated irqs: 0419worker1 # [ 0.000000] GICv3: using LPI property table @0x0000000044a00000420worker2 # [ 0.005383] CPU features: detected: Fine Grained Traps421lb # [ 0.003725] rcu: Hierarchical SRCU implementation.422client # [ 0.000000] GICv3: 256 SPIs implemented423worker1 # [ 0.000000] GICv3: CPU0: using allocated LPI pending table @0x0000000044a10000424lb # [ 0.003729] rcu: Max phase no-delay instances is 1000.425client # [ 0.000000] GICv3: 0 Extended SPIs implemented426worker2 # [ 0.005387] CPU features: detected: Generic authentication (architected QARMA5 algorithm)427client # [ 0.000000] Root IRQ handler: gic_handle_irq428lb # [ 0.004934] fsl-mc MSI: its@8080000 domain created429worker1 # [ 0.000000] rcu: srcu_init: Setting srcu_struct sizes based on contention.430worker2 # [ 0.005393] CPU features: detected: RCpc load-acquire (LDAPR)431lb # [ 0.005028] EFI services will not be available.432client # [ 0.000000] GICv3: GICv3 features: 16 PPIs, DirectLPI433lb # [ 0.005094] smp: Bringing up secondary CPUs ...434client # [ 0.000000] GICv3: GICD_CTLR.DS=1, SCR_EL3.FIQ=0435worker2 # [ 0.005396] CPU features: detected: LSE atomic instructions436worker1 # [ 0.000000] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns437lb # [ 0.005102] smp: Brought up 1 node, 1 CPU438worker2 # [ 0.005399] CPU features: detected: Privileged Access Never439client # [ 0.000000] GICv3: CPU0: found redistributor 0 region 0:0x00000000080a0000440lb # [ 0.005105] SMP: Total of 1 processors activated.441worker1 # [ 0.000000] arch_timer: cp15 timer running at 1000.00MHz (virt).442worker2 # [ 0.005401] CPU features: detected: PMUv3443client # [ 0.000000] ITS [mem 0x08080000-0x0809ffff]444lb # [ 0.005110] CPU: All CPU(s) started at EL1445worker2 # [ 0.005404] CPU features: detected: RAS Extension Support446lb # [ 0.005121] CPU features: detected: Branch Target Identification447worker1 # [ 0.000000] clocksource: arch_sys_counter: mask: 0x1fffffffffffffff max_cycles: 0x1cd42e4dffb, max_idle_ns: 881590591483 ns448client # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Devices @449e0000 (indirect, esz 8, psz 64K, shr 1)449worker2 # [ 0.005407] CPU features: detected: RASv1p1 Extension Support450lb # [ 0.005126] CPU features: detected: ARMv8.4 Translation Table Level451worker2 # [ 0.005409] CPU features: detected: Random Number Generator452worker1 # [ 0.000000] sched_clock: 61 bits at 1000MHz, resolution 1ns, wraps every 4398046511103ns453client # [ 0.000000] ITS@0x0000000008080000: allocated 8192 Interrupt Collections @449f0000 (flat, esz 8, psz 64K, shr 1)454worker1 # [ 0.000030] arm-pv: using stolen time PV455worker2 # [ 0.005412] CPU features: detected: Speculation barrier (SB)456lb # [ 0.005129] CPU features: detected: Instruction cache invalidation not required for I/D coherence457client # [ 0.000000] GICv3: using LPI property table @0x0000000044a00000458worker2 # [ 0.005415] CPU features: detected: Stage-2 Force Write-Back459worker1 # [ 0.000399] kfence: initialized - using 2097152 bytes for 255 objects at 0x(____ptrval____)-0x(____ptrval____)460lb # [ 0.005133] CPU features: detected: Data cache clean to the PoU not required for I/D coherence461client # [ 0.000000] GICv3: CPU0: using allocated LPI pending table @0x0000000044a10000462worker2 # [ 0.005418] CPU features: detected: TLB range maintenance instructions463worker1 # [ 0.000547] Console: colour dummy device 80x25464lb # [ 0.005136] CPU features: detected: Common not Private translations465worker1 # [ 0.000555] printk: legacy console [tty0] enabled466client # [ 0.000000] rcu: srcu_init: Setting srcu_struct sizes based on contention.467worker2 # [ 0.005422] CPU features: detected: Speculative Store Bypassing Safe (SSBS)468lb # [ 0.005139] CPU features: detected: CRC32 instructions469worker2 # [ 0.005461] alternatives: applying system-wide alternatives470client # [ 0.000000] clocksource: jiffies: mask: 0xffffffff max_cycles: 0xffffffff, max_idle_ns: 7645041785100000 ns471worker1 # [ 0.000763] Calibrating delay loop (skipped), value calculated using timer frequency.. 2000.00 BogoMIPS (lpj=4000000)472lb # [ 0.005142] CPU features: detected: Data cache clean to Point of Deep Persistence473worker2 # [ 0.008359] CPU features: detected: BBM Level 2 without TLB conflict abort474worker1 # [ 0.000770] pid_max: default: 32768 minimum: 301475client # [ 0.000000] arch_timer: cp15 timer running at 1000.00MHz (virt).476lb # [ 0.005146] CPU features: detected: Data cache clean to Point of Persistence477worker1 # [ 0.000862] LSM: initializing lsm=capability,landlock,yama,bpf,ima478lb # [ 0.005149] CPU features: detected: Data independent timing control (DIT)479worker1 # [ 0.001018] landlock: Up and running.480client # [ 0.000000] clocksource: arch_sys_counter: mask: 0x1fffffffffffffff max_cycles: 0x1cd42e4dffb, max_idle_ns: 881590591483 ns481worker2 # [ 0.008557] Memory: 1406544K/1572864K available (24448K kernel code, 7090K rwdata, 26348K rodata, 4736K init, 1107K bss, 123772K reserved, 32768K cma-reserved)482lb # [ 0.005152] CPU features: detected: E0PD483worker1 # [ 0.001022] Yama: becoming mindful.484worker2 # [ 0.008910] devtmpfs: initialized485worker1 # [ 0.001454] LSM support for eBPF active486lb # [ 0.005155] CPU features: detected: Enhanced Counter Virtualization487client # [ 0.000000] sched_clock: 61 bits at 1000MHz, resolution 1ns, wraps every 4398046511103ns488worker2 # [ 0.010674] posixtimers hash table entries: 512 (order: 1, 8192 bytes, linear)489client # [ 0.000028] arm-pv: using stolen time PV490worker1 # [ 0.001600] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)491lb # [ 0.005157] CPU features: detected: Enhanced Counter Virtualization (CNTPOFF)492worker2 # [ 0.010695] futex hash table entries: 256 (16384 bytes on 1 NUMA nodes, total 16 KiB, linear).493worker1 # [ 0.001632] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)494lb # [ 0.005161] CPU features: detected: Enhanced Virtualization Traps495client # [ 0.000383] kfence: initialized - using 2097152 bytes for 255 objects at 0x(____ptrval____)-0x(____ptrval____)496worker2 # [ 0.010882] 2G module region forced by RANDOMIZE_MODULE_REGION_FULL497worker1 # [ 0.002675] cacheinfo: Unable to detect cache hierarchy for CPU 0498client # [ 0.000543] Console: colour dummy device 80x25499lb # [ 0.005164] CPU features: detected: Fine Grained Traps500worker2 # [ 0.010887] 0 pages in range for non-PLT usage501client # [ 0.000550] printk: legacy console [tty0] enabled502worker1 # [ 0.003395] rcu: Hierarchical SRCU implementation.503lb # [ 0.005168] CPU features: detected: Generic authentication (architected QARMA5 algorithm)504worker1 # [ 0.003398] rcu: Max phase no-delay instances is 1000.505worker2 # [ 0.010888] 508288 pages in range for PLT usage506worker2 # [ 0.011001] pinctrl core: initialized pinctrl subsystem507lb # [ 0.005172] CPU features: detected: RCpc load-acquire (LDAPR)508worker1 # [ 0.004569] fsl-mc MSI: its@8080000 domain created509client # [ 0.000737] Calibrating delay loop (skipped), value calculated using timer frequency.. 2000.00 BogoMIPS (lpj=4000000)510worker2 # [ 0.011777] DMI not present or invalid.511lb # [ 0.005175] CPU features: detected: LSE atomic instructions512client # [ 0.000744] pid_max: default: 32768 minimum: 301513worker1 # [ 0.004659] EFI services will not be available.514worker2 # [ 0.015095] NET: Registered PF_NETLINK/PF_ROUTE protocol family515worker1 # [ 0.004736] smp: Bringing up secondary CPUs ...516lb # [ 0.005178] CPU features: detected: Privileged Access Never517client # [ 0.000812] LSM: initializing lsm=capability,landlock,yama,bpf,ima518worker1 # [ 0.004744] smp: Brought up 1 node, 1 CPU519client # [ 0.000930] landlock: Up and running.520lb # [ 0.005181] CPU features: detected: PMUv3521worker2 # [ 0.017526] DMA: preallocated 256 KiB GFP_KERNEL pool for atomic allocations522worker1 # [ 0.004747] SMP: Total of 1 processors activated.523client # [ 0.000932] Yama: becoming mindful.524lb # [ 0.005183] CPU features: detected: RAS Extension Support525worker1 # [ 0.004752] CPU: All CPU(s) started at EL1526client # [ 0.001356] LSM support for eBPF active527worker2 # [ 0.017724] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA pool for atomic allocations528lb # [ 0.005186] CPU features: detected: RASv1p1 Extension Support529worker1 # [ 0.004763] CPU features: detected: Branch Target Identification530client # [ 0.001499] Mount-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)531worker2 # [ 0.017920] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations532lb # [ 0.005189] CPU features: detected: Random Number Generator533worker1 # [ 0.004768] CPU features: detected: ARMv8.4 Translation Table Level534lb # [ 0.005191] CPU features: detected: Speculation barrier (SB)535client # [ 0.001535] Mountpoint-cache hash table entries: 4096 (order: 3, 32768 bytes, linear)536worker2 # [ 0.017940] audit: initializing netlink subsys (disabled)537lb # [ 0.005194] CPU features: detected: Stage-2 Force Write-Back538worker2 # [ 0.018535] thermal_sys: Registered thermal governor 'fair_share'539worker1 # [ 0.004771] CPU features: detected: Instruction cache invalidation not required for I/D coherence540client # [ 0.002663] cacheinfo: Unable to detect cache hierarchy for CPU 0541client # [ 0.003365] rcu: Hierarchical SRCU implementation.542worker2 # [ 0.018537] thermal_sys: Registered thermal governor 'bang_bang'543lb # [ 0.005197] CPU features: detected: TLB range maintenance instructions544worker1 # [ 0.004775] CPU features: detected: Data cache clean to the PoU not required for I/D coherence545client # [ 0.003368] rcu: Max phase no-delay instances is 1000.546worker2 # [ 0.018541] thermal_sys: Registered thermal governor 'step_wise'547lb # [ 0.005201] CPU features: detected: Speculative Store Bypassing Safe (SSBS)548worker1 # [ 0.004778] CPU features: detected: Common not Private translations549client # [ 0.004522] fsl-mc MSI: its@8080000 domain created550worker2 # [ 0.018546] thermal_sys: Registered thermal governor 'user_space'551lb # [ 0.005240] alternatives: applying system-wide alternatives552client # [ 0.004611] EFI services will not be available.553worker1 # [ 0.004781] CPU features: detected: CRC32 instructions554client # [ 0.004691] smp: Bringing up secondary CPUs ...555worker2 # [ 0.018549] thermal_sys: Registered thermal governor 'power_allocator'556lb # [ 0.008160] CPU features: detected: BBM Level 2 without TLB conflict abort557worker1 # [ 0.004784] CPU features: detected: Data cache clean to Point of Deep Persistence558client # [ 0.004699] smp: Brought up 1 node, 1 CPU559worker2 # [ 0.018574] audit: type=2000 audit(0.016:1): state=initialized audit_enabled=0 res=1560client # [ 0.004702] SMP: Total of 1 processors activated.561worker1 # [ 0.004788] CPU features: detected: Data cache clean to Point of Persistence562worker2 # [ 0.018582] cpuidle: using governor ladder563client # [ 0.004707] CPU: All CPU(s) started at EL1564worker2 # [ 0.018588] cpuidle: using governor menu565lb # [ 0.008313] Memory: 1406540K/1572864K available (24448K kernel code, 7090K rwdata, 26348K rodata, 4736K init, 1107K bss, 123764K reserved, 32768K cma-reserved)566worker1 # [ 0.004792] CPU features: detected: Data independent timing control (DIT)567client # [ 0.004718] CPU features: detected: Branch Target Identification568lb # [ 0.008690] devtmpfs: initialized569worker1 # [ 0.004795] CPU features: detected: E0PD570worker2 # [ 0.018805] hw-breakpoint: found 16 breakpoint and 4 watchpoint registers.571client # [ 0.004723] CPU features: detected: ARMv8.4 Translation Table Level572lb # [ 0.010538] posixtimers hash table entries: 512 (order: 1, 8192 bytes, linear)573worker1 # [ 0.004797] CPU features: detected: Enhanced Counter Virtualization574worker2 # [ 0.018821] ASID allocator initialised with 65536 entries575worker2 # [ 0.020092] Serial: AMBA PL011 UART driver576client # [ 0.004726] CPU features: detected: Instruction cache invalidation not required for I/D coherence577worker1 # [ 0.004800] CPU features: detected: Enhanced Counter Virtualization (CNTPOFF)578lb # [ 0.010559] futex hash table entries: 256 (16384 bytes on 1 NUMA nodes, total 16 KiB, linear).579worker2 # [ 0.025301] 9000000.pl011: ttyAMA0 at MMIO 0x9000000 (irq = 13, base_baud = 0) is a PL011 rev1580worker1 # [ 0.004803] CPU features: detected: Enhanced Virtualization Traps581lb # [ 0.010743] 2G module region forced by RANDOMIZE_MODULE_REGION_FULL582client # [ 0.004730] CPU features: detected: Data cache clean to the PoU not required for I/D coherence583worker2 # [ 0.025454] printk: console [ttyAMA0] enabled584worker1 # [ 0.004807] CPU features: detected: Fine Grained Traps585lb # [ 0.010747] 0 pages in range for non-PLT usage586client # [ 0.004733] CPU features: detected: Common not Private translations587lb # [ 0.010748] 508288 pages in range for PLT usage588worker1 # [ 0.004825] CPU features: detected: Generic authentication (architected QARMA5 algorithm)589client # [ 0.004736] CPU features: detected: CRC32 instructions590lb # [ 0.010853] pinctrl core: initialized pinctrl subsystem591worker1 # [ 0.004830] CPU features: detected: RCpc load-acquire (LDAPR)592client # [ 0.004739] CPU features: detected: Data cache clean to Point of Deep Persistence593worker1 # [ 0.004833] CPU features: detected: LSE atomic instructions594client # [ 0.004743] CPU features: detected: Data cache clean to Point of Persistence595worker1 # [ 0.004836] CPU features: detected: Privileged Access Never596worker1 # [ 0.004839] CPU features: detected: PMUv3597client # [ 0.004746] CPU features: detected: Data independent timing control (DIT)598client # [ 0.004749] CPU features: detected: E0PD599worker1 # [ 0.004842] CPU features: detected: RAS Extension Support600worker1 # [ 0.004844] CPU features: detected: RASv1p1 Extension Support601client # [ 0.004751] CPU features: detected: Enhanced Counter Virtualization602worker1 # [ 0.004847] CPU features: detected: Random Number Generator603client # [ 0.004754] CPU features: detected: Enhanced Counter Virtualization (CNTPOFF)604worker1 # [ 0.004850] CPU features: detected: Speculation barrier (SB)605client # [ 0.004758] CPU features: detected: Enhanced Virtualization Traps606worker1 # [ 0.004853] CPU features: detected: Stage-2 Force Write-Back607client # [ 0.004761] CPU features: detected: Fine Grained Traps608worker1 # [ 0.004856] CPU features: detected: TLB range maintenance instructions609worker2 # [ 0.143411] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages610client # [ 0.004765] CPU features: detected: Generic authentication (architected QARMA5 algorithm)611worker1 # [ 0.004861] CPU features: detected: Speculative Store Bypassing Safe (SSBS)612worker2 # [ 0.143429] HugeTLB: 0 KiB vmemmap can be freed for a 1.00 GiB page613worker1 # [ 0.004898] alternatives: applying system-wide alternatives614worker2 # [ 0.143435] HugeTLB: registered 32.0 MiB page size, pre-allocated 0 pages615lb # [ 0.011611] DMI not present or invalid.616worker1 # [ 0.007837] CPU features: detected: BBM Level 2 without TLB conflict abort617worker2 # [ 0.143439] HugeTLB: 0 KiB vmemmap can be freed for a 32.0 MiB page618lb # [ 0.014650] NET: Registered PF_NETLINK/PF_ROUTE protocol family619worker2 # [ 0.143444] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages620lb # [ 0.017043] DMA: preallocated 256 KiB GFP_KERNEL pool for atomic allocations621worker2 # [ 0.143448] HugeTLB: 0 KiB vmemmap can be freed for a 2.00 MiB page622worker1 # [ 0.007981] Memory: 1406568K/1572864K available (24448K kernel code, 7090K rwdata, 26348K rodata, 4736K init, 1107K bss, 123772K reserved, 32768K cma-reserved)623lb # [ 0.017235] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA pool for atomic allocations624worker1 # [ 0.008391] devtmpfs: initialized625worker2 # [ 0.143452] HugeTLB: registered 64.0 KiB page size, pre-allocated 0 pages626lb # [ 0.017429] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations627worker1 # [ 0.010195] posixtimers hash table entries: 512 (order: 1, 8192 bytes, linear)628worker2 # [ 0.143457] HugeTLB: 0 KiB vmemmap can be freed for a 64.0 KiB page629lb # [ 0.017451] audit: initializing netlink subsys (disabled)630worker1 # [ 0.010222] futex hash table entries: 256 (16384 bytes on 1 NUMA nodes, total 16 KiB, linear).631lb # [ 0.018010] thermal_sys: Registered thermal governor 'fair_share'632worker1 # [ 0.010400] 2G module region forced by RANDOMIZE_MODULE_REGION_FULL633lb # [ 0.018012] thermal_sys: Registered thermal governor 'bang_bang'634worker1 # [ 0.010404] 0 pages in range for non-PLT usage635worker2 # [ 0.150598] fbcon: Taking over console636worker1 # [ 0.010405] 508288 pages in range for PLT usage637lb # [ 0.018016] thermal_sys: Registered thermal governor 'step_wise'638worker2 # [ 0.150612] ACPI: Interpreter disabled.639worker1 # [ 0.010515] pinctrl core: initialized pinctrl subsystem640lb # [ 0.018021] thermal_sys: Registered thermal governor 'user_space'641client # [ 0.004770] CPU features: detected: RCpc load-acquire (LDAPR)642worker1 # [ 0.011271] DMI not present or invalid.643lb # [ 0.018023] thermal_sys: Registered thermal governor 'power_allocator'644client # [ 0.004773] CPU features: detected: LSE atomic instructions645worker1 # [ 0.014256] NET: Registered PF_NETLINK/PF_ROUTE protocol family646client # [ 0.004776] CPU features: detected: Privileged Access Never647lb # [ 0.018054] audit: type=2000 audit(0.016:1): state=initialized audit_enabled=0 res=1648worker1 # [ 0.016675] DMA: preallocated 256 KiB GFP_KERNEL pool for atomic allocations649client # [ 0.004778] CPU features: detected: PMUv3650lb # [ 0.018063] cpuidle: using governor ladder651client # [ 0.004781] CPU features: detected: RAS Extension Support652lb # [ 0.018069] cpuidle: using governor menu653client # [ 0.004784] CPU features: detected: RASv1p1 Extension Support654lb # [ 0.018247] hw-breakpoint: found 16 breakpoint and 4 watchpoint registers.655client # [ 0.004787] CPU features: detected: Random Number Generator656lb # [ 0.018261] ASID allocator initialised with 65536 entries657lb # [ 0.019416] Serial: AMBA PL011 UART driver658client # [ 0.004789] CPU features: detected: Speculation barrier (SB)659client # [ 0.004792] CPU features: detected: Stage-2 Force Write-Back660lb # [ 0.024627] 9000000.pl011: ttyAMA0 at MMIO 0x9000000 (irq = 13, base_baud = 0) is a PL011 rev1661client # [ 0.004795] CPU features: detected: TLB range maintenance instructions662lb # [ 0.024742] printk: console [ttyAMA0] enabled663client # [ 0.004800] CPU features: detected: Speculative Store Bypassing Safe (SSBS)664lb # [ 0.143621] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages665worker2 # [ 0.159506] iommu: Default domain type: Translated666client # [ 0.004837] alternatives: applying system-wide alternatives667lb # [ 0.143639] HugeTLB: 0 KiB vmemmap can be freed for a 1.00 GiB page668worker2 # [ 0.159516] iommu: DMA domain TLB invalidation policy: strict mode669client # [ 0.007735] CPU features: detected: BBM Level 2 without TLB conflict abort670worker2 # [ 0.159880] SCSI subsystem initialized671lb # [ 0.143644] HugeTLB: registered 32.0 MiB page size, pre-allocated 0 pages672lb # [ 0.143648] HugeTLB: 0 KiB vmemmap can be freed for a 32.0 MiB page673worker2 # [ 0.161782] usbcore: registered new interface driver usbfs674client # [ 0.007968] Memory: 1406588K/1572864K available (24448K kernel code, 7090K rwdata, 26348K rodata, 4736K init, 1107K bss, 123784K reserved, 32768K cma-reserved)675client # [ 0.008361] devtmpfs: initialized676lb # [ 0.143653] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages677worker2 # [ 0.161815] usbcore: registered new interface driver hub678worker1 # [ 0.016877] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA pool for atomic allocations679worker2 # [ 0.161833] usbcore: registered new device driver usb680lb # [ 0.143657] HugeTLB: 0 KiB vmemmap can be freed for a 2.00 MiB page681client # [ 0.010180] posixtimers hash table entries: 512 (order: 1, 8192 bytes, linear)682worker1 # [ 0.017076] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations683worker2 # [ 0.162092] pps_core: LinuxPPS API ver. 1 registered684lb # [ 0.143661] HugeTLB: registered 64.0 KiB page size, pre-allocated 0 pages685client # [ 0.010200] futex hash table entries: 256 (16384 bytes on 1 NUMA nodes, total 16 KiB, linear).686worker1 # [ 0.017097] audit: initializing netlink subsys (disabled)687worker2 # [ 0.162098] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>688lb # [ 0.143665] HugeTLB: 0 KiB vmemmap can be freed for a 64.0 KiB page689worker1 # [ 0.017648] thermal_sys: Registered thermal governor 'fair_share'690client # [ 0.010374] 2G module region forced by RANDOMIZE_MODULE_REGION_FULL691worker2 # [ 0.162108] PTP clock support registered692client # [ 0.010379] 0 pages in range for non-PLT usage693worker2 # [ 0.162154] EDAC MC: Ver: 3.0.0694worker1 # [ 0.017650] thermal_sys: Registered thermal governor 'bang_bang'695client # [ 0.010380] 508288 pages in range for PLT usage696worker2 # [ 0.166552] scmi_core: SCMI protocol bus registered697worker1 # [ 0.017654] thermal_sys: Registered thermal governor 'step_wise'698client # [ 0.010482] pinctrl core: initialized pinctrl subsystem699lb # [ 0.150835] fbcon: Taking over console700client # [ 0.011234] DMI not present or invalid.701worker1 # [ 0.017659] thermal_sys: Registered thermal governor 'user_space'702lb # [ 0.150849] ACPI: Interpreter disabled.703worker2 # [ 0.167497] FPGA manager framework704client # [ 0.014272] NET: Registered PF_NETLINK/PF_ROUTE protocol family705worker2 # [ 0.168479] vgaarb: loaded706worker1 # [ 0.017662] thermal_sys: Registered thermal governor 'power_allocator'707lb # [ 0.152700] iommu: Default domain type: Translated708client # [ 0.016662] DMA: preallocated 256 KiB GFP_KERNEL pool for atomic allocations709worker1 # [ 0.017684] audit: type=2000 audit(0.016:1): state=initialized audit_enabled=0 res=1710lb # [ 0.152709] iommu: DMA domain TLB invalidation policy: strict mode711worker2 # [ 0.169075] clocksource: Switched to clocksource arch_sys_counter712worker1 # [ 0.017693] cpuidle: using governor ladder713client # [ 0.016835] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA pool for atomic allocations714lb # [ 0.154285] SCSI subsystem initialized715worker1 # [ 0.017699] cpuidle: using governor menu716client # [ 0.017029] DMA: preallocated 256 KiB GFP_KERNEL|GFP_DMA32 pool for atomic allocations717worker1 # [ 0.017880] hw-breakpoint: found 16 breakpoint and 4 watchpoint registers.718client # [ 0.017075] audit: initializing netlink subsys (disabled)719worker1 # [ 0.017894] ASID allocator initialised with 65536 entries720worker2 # [ 0.171789] VFS: Disk quotas dquot_6.6.0721worker1 # [ 0.019065] Serial: AMBA PL011 UART driver722client # [ 0.017607] thermal_sys: Registered thermal governor 'fair_share'723worker2 # [ 0.171825] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)724client # [ 0.017610] thermal_sys: Registered thermal governor 'bang_bang'725worker1 # [ 0.024204] 9000000.pl011: ttyAMA0 at MMIO 0x9000000 (irq = 13, base_baud = 0) is a PL011 rev1726client # [ 0.017613] thermal_sys: Registered thermal governor 'step_wise'727worker1 # [ 0.024328] printk: console [ttyAMA0] enabled728client # [ 0.017618] thermal_sys: Registered thermal governor 'user_space'729worker1 # [ 0.143651] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages730client # [ 0.017621] thermal_sys: Registered thermal governor 'power_allocator'731worker2 # [ 0.175423] netfs: FS-Cache loaded732worker1 # [ 0.143669] HugeTLB: 0 KiB vmemmap can be freed for a 1.00 GiB page733client # [ 0.017643] audit: type=2000 audit(0.016:1): state=initialized audit_enabled=0 res=1734worker2 # [ 0.175540] pnp: PnP ACPI: disabled735lb # [ 0.159825] usbcore: registered new interface driver usbfs736client # [ 0.017652] cpuidle: using governor ladder737worker1 # [ 0.143674] HugeTLB: registered 32.0 MiB page size, pre-allocated 0 pages738lb # [ 0.159861] usbcore: registered new interface driver hub739client # [ 0.017658] cpuidle: using governor menu740worker1 # [ 0.143678] HugeTLB: 0 KiB vmemmap can be freed for a 32.0 MiB page741lb # [ 0.159879] usbcore: registered new device driver usb742client # [ 0.017846] hw-breakpoint: found 16 breakpoint and 4 watchpoint registers.743lb # [ 0.160157] pps_core: LinuxPPS API ver. 1 registered744worker1 # [ 0.143683] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages745client # [ 0.017861] ASID allocator initialised with 65536 entries746worker1 # [ 0.143687] HugeTLB: 0 KiB vmemmap can be freed for a 2.00 MiB page747client # [ 0.019008] Serial: AMBA PL011 UART driver748lb # [ 0.160163] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>749lb # [ 0.160173] PTP clock support registered750worker1 # [ 0.143691] HugeTLB: registered 64.0 KiB page size, pre-allocated 0 pages751client # [ 0.024105] 9000000.pl011: ttyAMA0 at MMIO 0x9000000 (irq = 13, base_baud = 0) is a PL011 rev1752lb # [ 0.160221] EDAC MC: Ver: 3.0.0753client # [ 0.024270] printk: console [ttyAMA0] enabled754worker1 # [ 0.143695] HugeTLB: 0 KiB vmemmap can be freed for a 64.0 KiB page755lb # [ 0.164674] scmi_core: SCMI protocol bus registered756worker2 # [ 0.179348] NET: Registered PF_INET protocol family757client # [ 0.143507] HugeTLB: registered 1.00 GiB page size, pre-allocated 0 pages758lb # [ 0.165606] FPGA manager framework759worker2 # [ 0.179597] IP idents hash table entries: 32768 (order: 6, 262144 bytes, linear)760lb # [ 0.166533] vgaarb: loaded761client # [ 0.143525] HugeTLB: 0 KiB vmemmap can be freed for a 1.00 GiB page762worker1 # [ 0.150921] fbcon: Taking over console763worker1 # [ 0.150935] ACPI: Interpreter disabled.764lb # [ 0.167163] clocksource: Switched to clocksource arch_sys_counter765client # [ 0.143530] HugeTLB: registered 32.0 MiB page size, pre-allocated 0 pages766worker1 # [ 0.152780] iommu: Default domain type: Translated767client # [ 0.143535] HugeTLB: 0 KiB vmemmap can be freed for a 32.0 MiB page768worker1 # [ 0.152789] iommu: DMA domain TLB invalidation policy: strict mode769client # [ 0.143539] HugeTLB: registered 2.00 MiB page size, pre-allocated 0 pages770worker1 # [ 0.154531] SCSI subsystem initialized771client # [ 0.143543] HugeTLB: 0 KiB vmemmap can be freed for a 2.00 MiB page772client # [ 0.143548] HugeTLB: registered 64.0 KiB page size, pre-allocated 0 pages773lb # [ 0.170483] VFS: Disk quotas dquot_6.6.0774client # [ 0.143552] HugeTLB: 0 KiB vmemmap can be freed for a 64.0 KiB page775lb # [ 0.170511] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)776client # [ 0.150993] fbcon: Taking over console777client # [ 0.151006] ACPI: Interpreter disabled.778lb # [ 0.174270] netfs: FS-Cache loaded779lb # [ 0.174387] pnp: PnP ACPI: disabled780worker1 # [ 0.161631] usbcore: registered new interface driver usbfs781worker1 # [ 0.161665] usbcore: registered new interface driver hub782worker1 # [ 0.161690] usbcore: registered new device driver usb783worker1 # [ 0.161962] pps_core: LinuxPPS API ver. 1 registered784lb # [ 0.178175] NET: Registered PF_INET protocol family785client # [ 0.158132] iommu: Default domain type: Translated786lb # [ 0.178431] IP idents hash table entries: 32768 (order: 6, 262144 bytes, linear)787worker1 # [ 0.161968] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>788worker1 # [ 0.161978] PTP clock support registered789client # [ 0.158141] iommu: DMA domain TLB invalidation policy: strict mode790worker1 # [ 0.162025] EDAC MC: Ver: 3.0.0791client # [ 0.159799] SCSI subsystem initialized792worker1 # [ 0.166592] scmi_core: SCMI protocol bus registered793client # [ 0.160462] usbcore: registered new interface driver usbfs794worker1 # [ 0.167545] FPGA manager framework795worker1 # [ 0.168542] vgaarb: loaded796client # [ 0.160494] usbcore: registered new interface driver hub797client # [ 0.160511] usbcore: registered new device driver usb798worker1 # [ 0.169181] clocksource: Switched to clocksource arch_sys_counter799client # [ 0.160784] pps_core: LinuxPPS API ver. 1 registered800client # [ 0.160790] pps_core: Software ver. 5.3.6 - Copyright 2005-2007 Rodolfo Giometti <giometti@linux.it>801worker1 # [ 0.171148] VFS: Disk quotas dquot_6.6.0802client # [ 0.160800] PTP clock support registered803client # [ 0.160854] EDAC MC: Ver: 3.0.0804worker1 # [ 0.171180] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)805client # [ 0.165492] scmi_core: SCMI protocol bus registered806client # [ 0.166476] FPGA manager framework807client # [ 0.167413] vgaarb: loaded808worker1 # [ 0.174868] netfs: FS-Cache loaded809client # [ 0.168030] clocksource: Switched to clocksource arch_sys_counter810worker1 # [ 0.174988] pnp: PnP ACPI: disabled811client # [ 0.170993] VFS: Disk quotas dquot_6.6.0812client # [ 0.171027] VFS: Dquot-cache hash table entries: 512 (order 0, 4096 bytes)813worker1 # [ 0.178924] NET: Registered PF_INET protocol family814client # [ 0.174622] netfs: FS-Cache loaded815client # [ 0.174740] pnp: PnP ACPI: disabled816worker1 # [ 0.179181] IP idents hash table entries: 32768 (order: 6, 262144 bytes, linear)817worker2 # [ 0.208146] tcp_listen_portaddr_hash hash table entries: 1024 (order: 2, 16384 bytes, linear)818client # [ 0.178551] NET: Registered PF_INET protocol family819worker2 # [ 0.208189] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear)820client # [ 0.178811] IP idents hash table entries: 32768 (order: 6, 262144 bytes, linear)821worker2 # [ 0.208215] TCP established hash table entries: 16384 (order: 5, 131072 bytes, linear)822worker2 # [ 0.208283] TCP bind hash table entries: 16384 (order: 7, 524288 bytes, linear)823worker2 # [ 0.208427] TCP: Hash tables configured (established 16384 bind 16384)824worker2 # [ 0.208502] MPTCP token hash table entries: 2048 (order: 4, 49152 bytes, linear)825worker2 # [ 0.208533] UDP hash table entries: 1024 (order: 4, 65536 bytes, linear)826worker2 # [ 0.208569] UDP-Lite hash table entries: 1024 (order: 4, 65536 bytes, linear)827worker2 # [ 0.208666] NET: Registered PF_UNIX/PF_LOCAL protocol family828worker2 # [ 0.208693] NET: Registered PF_XDP protocol family829worker2 # [ 0.208710] PCI: CLS 0 bytes, default 64830worker2 # [ 0.208950] Trying to unpack rootfs image as initramfs...831lb # [ 0.207465] tcp_listen_portaddr_hash hash table entries: 1024 (order: 2, 16384 bytes, linear)832lb # [ 0.207505] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear)833worker2 # [ 0.226631] kvm [1]: HYP mode not available834lb # [ 0.207532] TCP established hash table entries: 16384 (order: 5, 131072 bytes, linear)835lb # [ 0.207601] TCP bind hash table entries: 16384 (order: 7, 524288 bytes, linear)836lb # [ 0.207742] TCP: Hash tables configured (established 16384 bind 16384)837lb # [ 0.207817] MPTCP token hash table entries: 2048 (order: 4, 49152 bytes, linear)838lb # [ 0.207851] UDP hash table entries: 1024 (order: 4, 65536 bytes, linear)839lb # [ 0.207895] UDP-Lite hash table entries: 1024 (order: 4, 65536 bytes, linear)840lb # [ 0.208002] NET: Registered PF_UNIX/PF_LOCAL protocol family841lb # [ 0.208025] NET: Registered PF_XDP protocol family842lb # [ 0.208041] PCI: CLS 0 bytes, default 64843lb # [ 0.208276] Trying to unpack rootfs image as initramfs...844worker1 # [ 0.208749] tcp_listen_portaddr_hash hash table entries: 1024 (order: 2, 16384 bytes, linear)845worker1 # [ 0.208794] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear)846worker1 # [ 0.208840] TCP established hash table entries: 16384 (order: 5, 131072 bytes, linear)847worker1 # [ 0.208911] TCP bind hash table entries: 16384 (order: 7, 524288 bytes, linear)848worker1 # [ 0.209056] TCP: Hash tables configured (established 16384 bind 16384)849worker1 # [ 0.209137] MPTCP token hash table entries: 2048 (order: 4, 49152 bytes, linear)850worker1 # [ 0.209171] UDP hash table entries: 1024 (order: 4, 65536 bytes, linear)851client # [ 0.207757] tcp_listen_portaddr_hash hash table entries: 1024 (order: 2, 16384 bytes, linear)852lb # [ 0.224745] kvm [1]: HYP mode not available853client # [ 0.207803] Table-perturb hash table entries: 65536 (order: 6, 262144 bytes, linear)854client # [ 0.207831] TCP established hash table entries: 16384 (order: 5, 131072 bytes, linear)855client # [ 0.207910] TCP bind hash table entries: 16384 (order: 7, 524288 bytes, linear)856client # [ 0.211988] TCP: Hash tables configured (established 16384 bind 16384)857worker1 # [ 0.215585] UDP-Lite hash table entries: 1024 (order: 4, 65536 bytes, linear)858worker1 # [ 0.215714] NET: Registered PF_UNIX/PF_LOCAL protocol family859worker1 # [ 0.215739] NET: Registered PF_XDP protocol family860worker1 # [ 0.215757] PCI: CLS 0 bytes, default 64861worker1 # [ 0.216013] Trying to unpack rootfs image as initramfs...862client # [ 0.212858] MPTCP token hash table entries: 2048 (order: 4, 49152 bytes, linear)863client # [ 0.212897] UDP hash table entries: 1024 (order: 4, 65536 bytes, linear)864client # [ 0.212932] UDP-Lite hash table entries: 1024 (order: 4, 65536 bytes, linear)865client # [ 0.213031] NET: Registered PF_UNIX/PF_LOCAL protocol family866client # [ 0.213086] NET: Registered PF_XDP protocol family867client # [ 0.213106] PCI: CLS 0 bytes, default 64868client # [ 0.213336] Trying to unpack rootfs image as initramfs...869worker1 # [ 0.229201] kvm [1]: HYP mode not available870client # [ 0.225001] kvm [1]: HYP mode not available871worker2 # [ 0.317606] Initialise system trusted keyrings872worker2 # [ 0.318347] workingset: timestamp_bits=42 max_order=19 bucket_order=0873worker2 # [ 0.319585] squashfs: version 4.0 (2009/01/31) Phillip Lougher874worker2 # [ 0.320356] 9p: Installing v9fs 9p2000 file system support875lb # [ 0.315687] Initialise system trusted keyrings876lb # [ 0.316437] workingset: timestamp_bits=42 max_order=19 bucket_order=0877lb # [ 0.317730] squashfs: version 4.0 (2009/01/31) Phillip Lougher878lb # [ 0.318478] 9p: Installing v9fs 9p2000 file system support879worker2 # [ 0.340958] Key type asymmetric registered880worker2 # [ 0.340980] Asymmetric key parser 'x509' registered881worker2 # [ 0.341043] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 242)882worker2 # [ 0.349154] io scheduler mq-deadline registered883worker2 # [ 0.349174] io scheduler kyber registered884worker1 # [ 0.321710] Initialise system trusted keyrings885worker1 # [ 0.322450] workingset: timestamp_bits=42 max_order=19 bucket_order=0886client # [ 0.316557] Initialise system trusted keyrings887worker1 # [ 0.323729] squashfs: version 4.0 (2009/01/31) Phillip Lougher888lb # [ 0.339091] Key type asymmetric registered889client # [ 0.317338] workingset: timestamp_bits=42 max_order=19 bucket_order=0890lb # [ 0.339114] Asymmetric key parser 'x509' registered891worker1 # [ 0.324516] 9p: Installing v9fs 9p2000 file system support892client # [ 0.318652] squashfs: version 4.0 (2009/01/31) Phillip Lougher893client # [ 0.319399] 9p: Installing v9fs 9p2000 file system support894lb # [ 0.347222] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 242)895lb # [ 0.348270] io scheduler mq-deadline registered896lb # [ 0.348282] io scheduler kyber registered897worker2 # [ 0.361213] pl061_gpio 9030000.pl061: PL061 GPIO chip registered898worker2 # [ 0.361811] ledtrig-cpu: registered to indicate activity on CPUs899worker2 # [ 0.362161] pci-host-generic 4010000000.pcie: host bridge /pcie@10000000 ranges:900worker2 # [ 0.362179] pci-host-generic 4010000000.pcie: IO 0x003eff0000..0x003effffff -> 0x0000000000901worker2 # [ 0.362191] pci-host-generic 4010000000.pcie: MEM 0x0010000000..0x003efeffff -> 0x0010000000902worker2 # [ 0.362199] pci-host-generic 4010000000.pcie: MEM 0x8000000000..0xffffffffff -> 0x8000000000903worker2 # [ 0.362226] pci-host-generic 4010000000.pcie: Memory resource size exceeds max for 32 bits904lb # [ 0.359318] pl061_gpio 9030000.pl061: PL061 GPIO chip registered905worker1 # [ 0.345140] Key type asymmetric registered906worker2 # [ 0.362249] pci-host-generic 4010000000.pcie: ECAM at [mem 0x4010000000-0x401fffffff] for [bus 00-ff]907worker1 # [ 0.345163] Asymmetric key parser 'x509' registered908worker2 # [ 0.362320] pci-host-generic 4010000000.pcie: PCI host bridge to bus 0000:00909worker2 # [ 0.362330] pci_bus 0000:00: root bus resource [bus 00-ff]910client # [ 0.339944] Key type asymmetric registered911worker2 # [ 0.362337] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]912client # [ 0.339966] Asymmetric key parser 'x509' registered913worker2 # [ 0.362342] pci_bus 0000:00: root bus resource [mem 0x10000000-0x3efeffff]914lb # [ 0.360707] ledtrig-cpu: registered to indicate activity on CPUs915client # [ 0.340035] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 242)916worker2 # [ 0.362347] pci_bus 0000:00: root bus resource [mem 0x8000000000-0xffffffffff]917lb # [ 0.361096] pci-host-generic 4010000000.pcie: host bridge /pcie@10000000 ranges:918worker2 # [ 0.362457] pci 0000:00:00.0: [1b36:0008] type 00 class 0x060000 conventional PCI endpoint919lb # [ 0.361112] pci-host-generic 4010000000.pcie: IO 0x003eff0000..0x003effffff -> 0x0000000000920worker2 # [ 0.362885] pci 0000:00:01.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint921lb # [ 0.361124] pci-host-generic 4010000000.pcie: MEM 0x0010000000..0x003efeffff -> 0x0010000000922worker2 # [ 0.363067] pci 0000:00:01.0: BAR 0 [io 0x0000-0x001f]923lb # [ 0.361132] pci-host-generic 4010000000.pcie: MEM 0x8000000000..0xffffffffff -> 0x8000000000924worker2 # [ 0.363083] pci 0000:00:01.0: BAR 1 [mem 0x00000000-0x00000fff]925worker2 # [ 0.363112] pci 0000:00:01.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]926lb # [ 0.361153] pci-host-generic 4010000000.pcie: Memory resource size exceeds max for 32 bits927worker1 # [ 0.353257] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 242)928worker2 # [ 0.363127] pci 0000:00:01.0: ROM [mem 0x00000000-0x0003ffff pref]929worker1 # [ 0.354268] io scheduler mq-deadline registered930lb # [ 0.361176] pci-host-generic 4010000000.pcie: ECAM at [mem 0x4010000000-0x401fffffff] for [bus 00-ff]931worker1 # [ 0.354279] io scheduler kyber registered932worker2 # [ 0.363578] pci 0000:00:02.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint933client # [ 0.348187] io scheduler mq-deadline registered934lb # [ 0.361251] pci-host-generic 4010000000.pcie: PCI host bridge to bus 0000:00935worker2 # [ 0.363774] pci 0000:00:02.0: BAR 0 [io 0x0000-0x001f]936client # [ 0.348194] io scheduler kyber registered937lb # [ 0.361261] pci_bus 0000:00: root bus resource [bus 00-ff]938worker2 # [ 0.363789] pci 0000:00:02.0: BAR 1 [mem 0x00000000-0x00000fff]939lb # [ 0.361267] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]940worker2 # [ 0.363818] pci 0000:00:02.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]941lb # [ 0.361271] pci_bus 0000:00: root bus resource [mem 0x10000000-0x3efeffff]942worker2 # [ 0.364273] pci 0000:00:03.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint943lb # [ 0.361276] pci_bus 0000:00: root bus resource [mem 0x8000000000-0xffffffffff]944worker2 # [ 0.364455] pci 0000:00:03.0: BAR 0 [io 0x0000-0x003f]945lb # [ 0.361331] pci 0000:00:00.0: [1b36:0008] type 00 class 0x060000 conventional PCI endpoint946worker2 # [ 0.364470] pci 0000:00:03.0: BAR 1 [mem 0x00000000-0x00000fff]947worker2 # [ 0.364499] pci 0000:00:03.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]948lb # [ 0.361772] pci 0000:00:01.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint949lb # [ 0.361956] pci 0000:00:01.0: BAR 0 [io 0x0000-0x001f]950worker2 # [ 0.364954] pci 0000:00:04.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint951lb # [ 0.361972] pci 0000:00:01.0: BAR 1 [mem 0x00000000-0x00000fff]952worker2 # [ 0.365154] pci 0000:00:04.0: BAR 0 [io 0x0000-0x001f]953lb # [ 0.362001] pci 0000:00:01.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]954worker2 # [ 0.365171] pci 0000:00:04.0: BAR 1 [mem 0x00000000-0x00000fff]955lb # [ 0.362017] pci 0000:00:01.0: ROM [mem 0x00000000-0x0003ffff pref]956worker2 # [ 0.365202] pci 0000:00:04.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]957client # [ 0.356166] pl061_gpio 9030000.pl061: PL061 GPIO chip registered958lb # [ 0.362464] pci 0000:00:02.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint959worker2 # [ 0.365652] pci 0000:00:05.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint960lb # [ 0.362647] pci 0000:00:02.0: BAR 0 [io 0x0000-0x001f]961worker2 # [ 0.365830] pci 0000:00:05.0: BAR 0 [io 0x0000-0x001f]962lb # [ 0.362662] pci 0000:00:02.0: BAR 1 [mem 0x00000000-0x00000fff]963worker2 # [ 0.365845] pci 0000:00:05.0: BAR 1 [mem 0x00000000-0x00000fff]964worker1 # [ 0.365293] pl061_gpio 9030000.pl061: PL061 GPIO chip registered965lb # [ 0.362691] pci 0000:00:02.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]966worker2 # [ 0.365874] pci 0000:00:05.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]967client # [ 0.357468] ledtrig-cpu: registered to indicate activity on CPUs968lb # [ 0.363137] pci 0000:00:03.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint969worker2 # [ 0.366322] pci 0000:00:06.0: [1af4:1001] type 00 class 0x010000 conventional PCI endpoint970client # [ 0.357817] pci-host-generic 4010000000.pcie: host bridge /pcie@10000000 ranges:971worker2 # [ 0.366501] pci 0000:00:06.0: BAR 0 [io 0x0000-0x007f]972lb # [ 0.382973] pci 0000:00:03.0: BAR 0 [io 0x0000-0x003f]973worker2 # [ 0.366516] pci 0000:00:06.0: BAR 1 [mem 0x00000000-0x00000fff]974lb # [ 0.382993] pci 0000:00:03.0: BAR 1 [mem 0x00000000-0x00000fff]975client # [ 0.357835] pci-host-generic 4010000000.pcie: IO 0x003eff0000..0x003effffff -> 0x0000000000976worker1 # [ 0.366619] ledtrig-cpu: registered to indicate activity on CPUs977worker2 # [ 0.366544] pci 0000:00:06.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]978lb # [ 0.383022] pci 0000:00:03.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]979client # [ 0.357847] pci-host-generic 4010000000.pcie: MEM 0x0010000000..0x003efeffff -> 0x0010000000980worker1 # [ 0.366992] pci-host-generic 4010000000.pcie: host bridge /pcie@10000000 ranges:981worker2 # [ 0.366987] pci 0000:00:07.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint982client # [ 0.357856] pci-host-generic 4010000000.pcie: MEM 0x8000000000..0xffffffffff -> 0x8000000000983worker2 # [ 0.367166] pci 0000:00:07.0: BAR 0 [io 0x0000-0x001f]984worker1 # [ 0.367009] pci-host-generic 4010000000.pcie: IO 0x003eff0000..0x003effffff -> 0x0000000000985worker2 # [ 0.367181] pci 0000:00:07.0: BAR 1 [mem 0x00000000-0x00000fff]986client # [ 0.357883] pci-host-generic 4010000000.pcie: Memory resource size exceeds max for 32 bits987worker1 # [ 0.367020] pci-host-generic 4010000000.pcie: MEM 0x0010000000..0x003efeffff -> 0x0010000000988lb # [ 0.385542] pci 0000:00:04.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint989worker2 # [ 0.367209] pci 0000:00:07.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]990client # [ 0.357906] pci-host-generic 4010000000.pcie: ECAM at [mem 0x4010000000-0x401fffffff] for [bus 00-ff]991lb # [ 0.385731] pci 0000:00:04.0: BAR 0 [io 0x0000-0x001f]992worker1 # [ 0.367029] pci-host-generic 4010000000.pcie: MEM 0x8000000000..0xffffffffff -> 0x8000000000993worker2 # [ 0.367225] pci 0000:00:07.0: ROM [mem 0x00000000-0x0003ffff pref]994client # [ 0.358006] pci-host-generic 4010000000.pcie: PCI host bridge to bus 0000:00995lb # [ 0.385746] pci 0000:00:04.0: BAR 1 [mem 0x00000000-0x00000fff]996worker2 # [ 0.367698] pci 0000:00:08.0: [1af4:1050] type 00 class 0x038000 conventional PCI endpoint997worker1 # [ 0.367048] pci-host-generic 4010000000.pcie: Memory resource size exceeds max for 32 bits998client # [ 0.358015] pci_bus 0000:00: root bus resource [bus 00-ff]999lb # [ 0.385778] pci 0000:00:04.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1000worker2 # [ 0.367890] pci 0000:00:08.0: BAR 1 [mem 0x00000000-0x00000fff]1001client # [ 0.358022] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]1002worker1 # [ 0.367071] pci-host-generic 4010000000.pcie: ECAM at [mem 0x4010000000-0x401fffffff] for [bus 00-ff]1003lb # [ 0.386266] pci 0000:00:05.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint1004worker2 # [ 0.367923] pci 0000:00:08.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1005client # [ 0.358027] pci_bus 0000:00: root bus resource [mem 0x10000000-0x3efeffff]1006worker1 # [ 0.367146] pci-host-generic 4010000000.pcie: PCI host bridge to bus 0000:001007lb # [ 0.386447] pci 0000:00:05.0: BAR 0 [io 0x0000-0x001f]1008worker2 # [ 0.368377] pci 0000:00:09.0: [1af4:1052] type 00 class 0x090000 conventional PCI endpoint1009worker1 # [ 0.367156] pci_bus 0000:00: root bus resource [bus 00-ff]1010client # [ 0.358032] pci_bus 0000:00: root bus resource [mem 0x8000000000-0xffffffffff]1011lb # [ 0.386463] pci 0000:00:05.0: BAR 1 [mem 0x00000000-0x00000fff]1012worker2 # [ 0.368560] pci 0000:00:09.0: BAR 1 [mem 0x00000000-0x00000fff]1013worker1 # [ 0.367162] pci_bus 0000:00: root bus resource [io 0x0000-0xffff]1014lb # [ 0.386492] pci 0000:00:05.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1015client # [ 0.358088] pci 0000:00:00.0: [1b36:0008] type 00 class 0x060000 conventional PCI endpoint1016worker2 # [ 0.368588] pci 0000:00:09.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1017worker1 # [ 0.367167] pci_bus 0000:00: root bus resource [mem 0x10000000-0x3efeffff]1018lb # [ 0.386935] pci 0000:00:06.0: [1af4:1001] type 00 class 0x010000 conventional PCI endpoint1019client # [ 0.358517] pci 0000:00:01.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint1020worker2 # [ 0.368985] pci 0000:00:0a.0: [8086:24cd] type 00 class 0x0c0320 conventional PCI endpoint1021worker1 # [ 0.367172] pci_bus 0000:00: root bus resource [mem 0x8000000000-0xffffffffff]1022lb # [ 0.387116] pci 0000:00:06.0: BAR 0 [io 0x0000-0x007f]1023client # [ 0.358701] pci 0000:00:01.0: BAR 0 [io 0x0000-0x001f]1024lb # [ 0.387131] pci 0000:00:06.0: BAR 1 [mem 0x00000000-0x00000fff]1025client # [ 0.358717] pci 0000:00:01.0: BAR 1 [mem 0x00000000-0x00000fff]1026worker1 # [ 0.367256] pci 0000:00:00.0: [1b36:0008] type 00 class 0x060000 conventional PCI endpoint1027lb # [ 0.387159] pci 0000:00:06.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1028client # [ 0.358747] pci 0000:00:01.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1029worker1 # [ 0.367688] pci 0000:00:01.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint1030client # [ 0.358762] pci 0000:00:01.0: ROM [mem 0x00000000-0x0003ffff pref]1031worker1 # [ 0.367874] pci 0000:00:01.0: BAR 0 [io 0x0000-0x001f]1032client # [ 0.359217] pci 0000:00:02.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint1033worker1 # [ 0.367890] pci 0000:00:01.0: BAR 1 [mem 0x00000000-0x00000fff]1034client # [ 0.359400] pci 0000:00:02.0: BAR 0 [io 0x0000-0x001f]1035worker1 # [ 0.367919] pci 0000:00:01.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1036client # [ 0.359415] pci 0000:00:02.0: BAR 1 [mem 0x00000000-0x00000fff]1037worker1 # [ 0.367935] pci 0000:00:01.0: ROM [mem 0x00000000-0x0003ffff pref]1038client # [ 0.359445] pci 0000:00:02.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1039worker1 # [ 0.368390] pci 0000:00:02.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint1040client # [ 0.359892] pci 0000:00:03.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint1041worker1 # [ 0.368571] pci 0000:00:02.0: BAR 0 [io 0x0000-0x001f]1042worker1 # [ 0.368586] pci 0000:00:02.0: BAR 1 [mem 0x00000000-0x00000fff]1043worker1 # [ 0.368615] pci 0000:00:02.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1044worker1 # [ 0.369071] pci 0000:00:03.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint1045lb # [ 0.399647] pci 0000:00:07.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint1046lb # [ 0.399836] pci 0000:00:07.0: BAR 0 [io 0x0000-0x001f]1047worker1 # [ 0.388798] pci 0000:00:03.0: BAR 0 [io 0x0000-0x003f]1048lb # [ 0.399851] pci 0000:00:07.0: BAR 1 [mem 0x00000000-0x00000fff]1049worker1 # [ 0.388824] pci 0000:00:03.0: BAR 1 [mem 0x00000000-0x00000fff]1050lb # [ 0.399891] pci 0000:00:07.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1051worker1 # [ 0.388853] pci 0000:00:03.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1052lb # [ 0.399907] pci 0000:00:07.0: ROM [mem 0x00000000-0x0003ffff pref]1053worker2 # [ 0.417206] pci 0000:00:0a.0: BAR 0 [mem 0x00000000-0x00000fff]1054lb # [ 0.400358] pci 0000:00:08.0: [1af4:1050] type 00 class 0x038000 conventional PCI endpoint1055worker2 # [ 0.417487] pci 0000:00:0b.0: [1af4:1003] type 00 class 0x078000 conventional PCI endpoint1056lb # [ 0.400544] pci 0000:00:08.0: BAR 1 [mem 0x00000000-0x00000fff]1057worker1 # [ 0.391479] pci 0000:00:04.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint1058worker2 # [ 0.417781] pci 0000:00:0b.0: BAR 0 [io 0x0000-0x003f]1059lb # [ 0.400573] pci 0000:00:08.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1060worker1 # [ 0.391663] pci 0000:00:04.0: BAR 0 [io 0x0000-0x001f]1061worker2 # [ 0.417798] pci 0000:00:0b.0: BAR 1 [mem 0x00000000-0x00000fff]1062lb # [ 0.401018] pci 0000:00:09.0: [1af4:1052] type 00 class 0x090000 conventional PCI endpoint1063worker1 # [ 0.391678] pci 0000:00:04.0: BAR 1 [mem 0x00000000-0x00000fff]1064worker2 # [ 0.417828] pci 0000:00:0b.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1065lb # [ 0.401203] pci 0000:00:09.0: BAR 1 [mem 0x00000000-0x00000fff]1066worker1 # [ 0.391710] pci 0000:00:04.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1067worker2 # [ 0.418291] pci 0000:00:0c.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint1068lb # [ 0.401232] pci 0000:00:09.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1069worker1 # [ 0.392162] pci 0000:00:05.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint1070worker2 # [ 0.418475] pci 0000:00:0c.0: BAR 0 [io 0x0000-0x001f]1071worker1 # [ 0.392344] pci 0000:00:05.0: BAR 0 [io 0x0000-0x001f]1072lb # [ 0.401633] pci 0000:00:0a.0: [8086:24cd] type 00 class 0x0c0320 conventional PCI endpoint1073worker2 # [ 0.418491] pci 0000:00:0c.0: BAR 1 [mem 0x00000000-0x00000fff]1074worker1 # [ 0.392360] pci 0000:00:05.0: BAR 1 [mem 0x00000000-0x00000fff]1075lb # [ 0.401813] pci 0000:00:0a.0: BAR 0 [mem 0x00000000-0x00000fff]1076worker2 # [ 0.418521] pci 0000:00:0c.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1077worker1 # [ 0.392389] pci 0000:00:05.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1078lb # [ 0.402054] pci 0000:00:0b.0: [1af4:1003] type 00 class 0x078000 conventional PCI endpoint1079client # [ 0.384124] pci 0000:00:03.0: BAR 0 [io 0x0000-0x003f]1080worker2 # [ 0.419105] pci 0000:00:01.0: ROM [mem 0x10000000-0x1003ffff pref]: assigned1081lb # [ 0.402324] pci 0000:00:0b.0: BAR 0 [io 0x0000-0x003f]1082worker1 # [ 0.392846] pci 0000:00:06.0: [1af4:1001] type 00 class 0x010000 conventional PCI endpoint1083client # [ 0.384152] pci 0000:00:03.0: BAR 1 [mem 0x00000000-0x00000fff]1084worker2 # [ 0.419116] pci 0000:00:07.0: ROM [mem 0x10040000-0x1007ffff pref]: assigned1085lb # [ 0.402342] pci 0000:00:0b.0: BAR 1 [mem 0x00000000-0x00000fff]1086worker1 # [ 0.393029] pci 0000:00:06.0: BAR 0 [io 0x0000-0x007f]1087client # [ 0.384181] pci 0000:00:03.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1088worker2 # [ 0.419121] pci 0000:00:01.0: BAR 4 [mem 0x8000000000-0x8000003fff 64bit pref]: assigned1089worker1 # [ 0.393045] pci 0000:00:06.0: BAR 1 [mem 0x00000000-0x00000fff]1090lb # [ 0.402370] pci 0000:00:0b.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1091client # [ 0.384666] pci 0000:00:04.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint1092worker2 # [ 0.419166] pci 0000:00:02.0: BAR 4 [mem 0x8000004000-0x8000007fff 64bit pref]: assigned1093worker1 # [ 0.393073] pci 0000:00:06.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1094client # [ 0.384850] pci 0000:00:04.0: BAR 0 [io 0x0000-0x001f]1095lb # [ 0.402819] pci 0000:00:0c.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint1096lb # [ 0.402999] pci 0000:00:0c.0: BAR 0 [io 0x0000-0x001f]1097client # [ 0.384865] pci 0000:00:04.0: BAR 1 [mem 0x00000000-0x00000fff]1098worker2 # [ 0.419212] pci 0000:00:03.0: BAR 4 [mem 0x8000008000-0x800000bfff 64bit pref]: assigned1099lb # [ 0.403014] pci 0000:00:0c.0: BAR 1 [mem 0x00000000-0x00000fff]1100client # [ 0.384897] pci 0000:00:04.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1101worker2 # [ 0.419259] pci 0000:00:04.0: BAR 4 [mem 0x800000c000-0x800000ffff 64bit pref]: assigned1102lb # [ 0.403042] pci 0000:00:0c.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1103client # [ 0.385356] pci 0000:00:05.0: [1af4:1009] type 00 class 0x000200 conventional PCI endpoint1104worker2 # [ 0.419306] pci 0000:00:05.0: BAR 4 [mem 0x8000010000-0x8000013fff 64bit pref]: assigned1105client # [ 0.385537] pci 0000:00:05.0: BAR 0 [io 0x0000-0x001f]1106worker2 # [ 0.419354] pci 0000:00:06.0: BAR 4 [mem 0x8000014000-0x8000017fff 64bit pref]: assigned1107client # [ 0.385553] pci 0000:00:05.0: BAR 1 [mem 0x00000000-0x00000fff]1108client # [ 0.385582] pci 0000:00:05.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1109worker2 # [ 0.419401] pci 0000:00:07.0: BAR 4 [mem 0x8000018000-0x800001bfff 64bit pref]: assigned1110client # [ 0.386039] pci 0000:00:06.0: [1af4:1001] type 00 class 0x010000 conventional PCI endpoint1111worker2 # [ 0.419450] pci 0000:00:08.0: BAR 4 [mem 0x800001c000-0x800001ffff 64bit pref]: assigned1112client # [ 0.386221] pci 0000:00:06.0: BAR 0 [io 0x0000-0x007f]1113worker2 # [ 0.419497] pci 0000:00:09.0: BAR 4 [mem 0x8000020000-0x8000023fff 64bit pref]: assigned1114client # [ 0.386236] pci 0000:00:06.0: BAR 1 [mem 0x00000000-0x00000fff]1115worker2 # [ 0.419546] pci 0000:00:0b.0: BAR 4 [mem 0x8000024000-0x8000027fff 64bit pref]: assigned1116client # [ 0.386265] pci 0000:00:06.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1117worker2 # [ 0.419606] pci 0000:00:0c.0: BAR 4 [mem 0x8000028000-0x800002bfff 64bit pref]: assigned1118client # [ 0.386746] pci 0000:00:07.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint1119worker2 # [ 0.419651] pci 0000:00:01.0: BAR 1 [mem 0x10080000-0x10080fff]: assigned1120client # [ 0.386930] pci 0000:00:07.0: BAR 0 [io 0x0000-0x001f]1121client # [ 0.386945] pci 0000:00:07.0: BAR 1 [mem 0x00000000-0x00000fff]1122worker2 # [ 0.419673] pci 0000:00:02.0: BAR 1 [mem 0x10081000-0x10081fff]: assigned1123lb # [ 0.423646] pci 0000:00:01.0: ROM [mem 0x10000000-0x1003ffff pref]: assigned1124client # [ 0.386974] pci 0000:00:07.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1125worker2 # [ 0.419694] pci 0000:00:03.0: BAR 1 [mem 0x10082000-0x10082fff]: assigned1126lb # [ 0.423662] pci 0000:00:07.0: ROM [mem 0x10040000-0x1007ffff pref]: assigned1127client # [ 0.386989] pci 0000:00:07.0: ROM [mem 0x00000000-0x0003ffff pref]1128worker2 # [ 0.419725] pci 0000:00:04.0: BAR 1 [mem 0x10083000-0x10083fff]: assigned1129lb # [ 0.423668] pci 0000:00:01.0: BAR 4 [mem 0x8000000000-0x8000003fff 64bit pref]: assigned1130worker2 # [ 0.419750] pci 0000:00:05.0: BAR 1 [mem 0x10084000-0x10084fff]: assigned1131client # [ 0.387440] pci 0000:00:08.0: [1af4:1050] type 00 class 0x038000 conventional PCI endpoint1132lb # [ 0.423714] pci 0000:00:02.0: BAR 4 [mem 0x8000004000-0x8000007fff 64bit pref]: assigned1133client # [ 0.387625] pci 0000:00:08.0: BAR 1 [mem 0x00000000-0x00000fff]1134worker2 # [ 0.419772] pci 0000:00:06.0: BAR 1 [mem 0x10085000-0x10085fff]: assigned1135lb # [ 0.423761] pci 0000:00:03.0: BAR 4 [mem 0x8000008000-0x800000bfff 64bit pref]: assigned1136client # [ 0.387654] pci 0000:00:08.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1137worker2 # [ 0.419794] pci 0000:00:07.0: BAR 1 [mem 0x10086000-0x10086fff]: assigned1138worker2 # [ 0.419816] pci 0000:00:08.0: BAR 1 [mem 0x10087000-0x10087fff]: assigned1139lb # [ 0.423807] pci 0000:00:04.0: BAR 4 [mem 0x800000c000-0x800000ffff 64bit pref]: assigned1140client # [ 0.388113] pci 0000:00:09.0: [1af4:1052] type 00 class 0x090000 conventional PCI endpoint1141worker1 # [ 0.409593] pci 0000:00:07.0: [1af4:1000] type 00 class 0x020000 conventional PCI endpoint1142worker2 # [ 0.419838] pci 0000:00:09.0: BAR 1 [mem 0x10088000-0x10088fff]: assigned1143client # [ 0.388298] pci 0000:00:09.0: BAR 1 [mem 0x00000000-0x00000fff]1144lb # [ 0.423866] pci 0000:00:05.0: BAR 4 [mem 0x8000010000-0x8000013fff 64bit pref]: assigned1145worker1 # [ 0.409791] pci 0000:00:07.0: BAR 0 [io 0x0000-0x001f]1146worker2 # [ 0.419861] pci 0000:00:0a.0: BAR 0 [mem 0x10089000-0x10089fff]: assigned1147client # [ 0.388327] pci 0000:00:09.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1148worker1 # [ 0.409808] pci 0000:00:07.0: BAR 1 [mem 0x00000000-0x00000fff]1149lb # [ 0.423912] pci 0000:00:06.0: BAR 4 [mem 0x8000014000-0x8000017fff 64bit pref]: assigned1150worker2 # [ 0.419884] pci 0000:00:0b.0: BAR 1 [mem 0x1008a000-0x1008afff]: assigned1151client # [ 0.388711] pci 0000:00:0a.0: [8086:24cd] type 00 class 0x0c0320 conventional PCI endpoint1152worker1 # [ 0.409836] pci 0000:00:07.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1153lb # [ 0.423958] pci 0000:00:07.0: BAR 4 [mem 0x8000018000-0x800001bfff 64bit pref]: assigned1154worker2 # [ 0.419906] pci 0000:00:0c.0: BAR 1 [mem 0x1008b000-0x1008bfff]: assigned1155client # [ 0.388890] pci 0000:00:0a.0: BAR 0 [mem 0x00000000-0x00000fff]1156worker1 # [ 0.409853] pci 0000:00:07.0: ROM [mem 0x00000000-0x0003ffff pref]1157worker2 # [ 0.419928] pci 0000:00:06.0: BAR 0 [io 0x1000-0x107f]: assigned1158lb # [ 0.424004] pci 0000:00:08.0: BAR 4 [mem 0x800001c000-0x800001ffff 64bit pref]: assigned1159client # [ 0.389137] pci 0000:00:0b.0: [1af4:1003] type 00 class 0x078000 conventional PCI endpoint1160worker1 # [ 0.410357] pci 0000:00:08.0: [1af4:1050] type 00 class 0x038000 conventional PCI endpoint1161worker2 # [ 0.419949] pci 0000:00:03.0: BAR 0 [io 0x1080-0x10bf]: assigned1162client # [ 0.389414] pci 0000:00:0b.0: BAR 0 [io 0x0000-0x003f]1163lb # [ 0.424050] pci 0000:00:09.0: BAR 4 [mem 0x8000020000-0x8000023fff 64bit pref]: assigned1164worker1 # [ 0.410544] pci 0000:00:08.0: BAR 1 [mem 0x00000000-0x00000fff]1165worker2 # [ 0.419969] pci 0000:00:0b.0: BAR 0 [io 0x10c0-0x10ff]: assigned1166client # [ 0.389431] pci 0000:00:0b.0: BAR 1 [mem 0x00000000-0x00000fff]1167lb # [ 0.424096] pci 0000:00:0b.0: BAR 4 [mem 0x8000024000-0x8000027fff 64bit pref]: assigned1168worker2 # [ 0.419990] pci 0000:00:01.0: BAR 0 [io 0x1100-0x111f]: assigned1169worker1 # [ 0.410574] pci 0000:00:08.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1170client # [ 0.389460] pci 0000:00:0b.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1171lb # [ 0.424189] pci 0000:00:0c.0: BAR 4 [mem 0x8000028000-0x800002bfff 64bit pref]: assigned1172worker2 # [ 0.420012] pci 0000:00:02.0: BAR 0 [io 0x1120-0x113f]: assigned1173worker1 # [ 0.411024] pci 0000:00:09.0: [1af4:1052] type 00 class 0x090000 conventional PCI endpoint1174client # [ 0.389919] pci 0000:00:0c.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint1175worker2 # [ 0.420033] pci 0000:00:04.0: BAR 0 [io 0x1140-0x115f]: assigned1176lb # [ 0.424235] pci 0000:00:01.0: BAR 1 [mem 0x10080000-0x10080fff]: assigned1177worker1 # [ 0.411212] pci 0000:00:09.0: BAR 1 [mem 0x00000000-0x00000fff]1178client # [ 0.390111] pci 0000:00:0c.0: BAR 0 [io 0x0000-0x001f]1179worker2 # [ 0.420054] pci 0000:00:05.0: BAR 0 [io 0x1160-0x117f]: assigned1180lb # [ 0.424256] pci 0000:00:02.0: BAR 1 [mem 0x10081000-0x10081fff]: assigned1181client # [ 0.390127] pci 0000:00:0c.0: BAR 1 [mem 0x00000000-0x00000fff]1182worker1 # [ 0.411241] pci 0000:00:09.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1183worker2 # [ 0.420077] pci 0000:00:07.0: BAR 0 [io 0x1180-0x119f]: assigned1184lb # [ 0.424277] pci 0000:00:03.0: BAR 1 [mem 0x10082000-0x10082fff]: assigned1185client # [ 0.390156] pci 0000:00:0c.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1186worker1 # [ 0.411639] pci 0000:00:0a.0: [8086:24cd] type 00 class 0x0c0320 conventional PCI endpoint1187worker2 # [ 0.420098] pci 0000:00:0c.0: BAR 0 [io 0x11a0-0x11bf]: assigned1188lb # [ 0.424298] pci 0000:00:04.0: BAR 1 [mem 0x10083000-0x10083fff]: assigned1189worker1 # [ 0.411820] pci 0000:00:0a.0: BAR 0 [mem 0x00000000-0x00000fff]1190worker2 # [ 0.420124] pci_bus 0000:00: resource 4 [io 0x0000-0xffff]1191client # [ 0.390727] pci 0000:00:01.0: ROM [mem 0x10000000-0x1003ffff pref]: assigned1192lb # [ 0.424322] pci 0000:00:05.0: BAR 1 [mem 0x10084000-0x10084fff]: assigned1193client # [ 0.390738] pci 0000:00:07.0: ROM [mem 0x10040000-0x1007ffff pref]: assigned1194worker2 # [ 0.420134] pci_bus 0000:00: resource 5 [mem 0x10000000-0x3efeffff]1195worker1 # [ 0.412066] pci 0000:00:0b.0: [1af4:1003] type 00 class 0x078000 conventional PCI endpoint1196lb # [ 0.424343] pci 0000:00:06.0: BAR 1 [mem 0x10085000-0x10085fff]: assigned1197worker1 # [ 0.412340] pci 0000:00:0b.0: BAR 0 [io 0x0000-0x003f]1198worker2 # [ 0.420139] pci_bus 0000:00: resource 6 [mem 0x8000000000-0xffffffffff]1199client # [ 0.390744] pci 0000:00:01.0: BAR 4 [mem 0x8000000000-0x8000003fff 64bit pref]: assigned1200lb # [ 0.424365] pci 0000:00:07.0: BAR 1 [mem 0x10086000-0x10086fff]: assigned1201worker2 # [ 0.421318] pci 0000:00:0a.0: enabling device (0000 -> 0002)1202worker1 # [ 0.412357] pci 0000:00:0b.0: BAR 1 [mem 0x00000000-0x00000fff]1203client # [ 0.390789] pci 0000:00:02.0: BAR 4 [mem 0x8000004000-0x8000007fff 64bit pref]: assigned1204lb # [ 0.424386] pci 0000:00:08.0: BAR 1 [mem 0x10087000-0x10087fff]: assigned1205worker1 # [ 0.412386] pci 0000:00:0b.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1206lb # [ 0.424408] pci 0000:00:09.0: BAR 1 [mem 0x10088000-0x10088fff]: assigned1207client # [ 0.390835] pci 0000:00:03.0: BAR 4 [mem 0x8000008000-0x800000bfff 64bit pref]: assigned1208worker1 # [ 0.412848] pci 0000:00:0c.0: [1af4:1005] type 00 class 0x00ff00 conventional PCI endpoint1209lb # [ 0.424429] pci 0000:00:0a.0: BAR 0 [mem 0x10089000-0x10089fff]: assigned1210client # [ 0.390881] pci 0000:00:04.0: BAR 4 [mem 0x800000c000-0x800000ffff 64bit pref]: assigned1211worker1 # [ 0.413030] pci 0000:00:0c.0: BAR 0 [io 0x0000-0x001f]1212lb # [ 0.424451] pci 0000:00:0b.0: BAR 1 [mem 0x1008a000-0x1008afff]: assigned1213worker1 # [ 0.413046] pci 0000:00:0c.0: BAR 1 [mem 0x00000000-0x00000fff]1214client # [ 0.390926] pci 0000:00:05.0: BAR 4 [mem 0x8000010000-0x8000013fff 64bit pref]: assigned1215lb # [ 0.424474] pci 0000:00:0c.0: BAR 1 [mem 0x1008b000-0x1008bfff]: assigned1216worker1 # [ 0.413075] pci 0000:00:0c.0: BAR 4 [mem 0x00000000-0x00003fff 64bit pref]1217client # [ 0.390971] pci 0000:00:06.0: BAR 4 [mem 0x8000014000-0x8000017fff 64bit pref]: assigned1218lb # [ 0.424495] pci 0000:00:06.0: BAR 0 [io 0x1000-0x107f]: assigned1219worker1 # [ 0.413669] pci 0000:00:01.0: ROM [mem 0x10000000-0x1003ffff pref]: assigned1220client # [ 0.391017] pci 0000:00:07.0: BAR 4 [mem 0x8000018000-0x800001bfff 64bit pref]: assigned1221lb # [ 0.424516] pci 0000:00:03.0: BAR 0 [io 0x1080-0x10bf]: assigned1222worker1 # [ 0.413680] pci 0000:00:07.0: ROM [mem 0x10040000-0x1007ffff pref]: assigned1223lb # [ 0.424537] pci 0000:00:0b.0: BAR 0 [io 0x10c0-0x10ff]: assigned1224client # [ 0.391062] pci 0000:00:08.0: BAR 4 [mem 0x800001c000-0x800001ffff 64bit pref]: assigned1225worker1 # [ 0.413686] pci 0000:00:01.0: BAR 4 [mem 0x8000000000-0x8000003fff 64bit pref]: assigned1226lb # [ 0.424557] pci 0000:00:01.0: BAR 0 [io 0x1100-0x111f]: assigned1227client # [ 0.391107] pci 0000:00:09.0: BAR 4 [mem 0x8000020000-0x8000023fff 64bit pref]: assigned1228lb # [ 0.424578] pci 0000:00:02.0: BAR 0 [io 0x1120-0x113f]: assigned1229worker1 # [ 0.413731] pci 0000:00:02.0: BAR 4 [mem 0x8000004000-0x8000007fff 64bit pref]: assigned1230client # [ 0.391153] pci 0000:00:0b.0: BAR 4 [mem 0x8000024000-0x8000027fff 64bit pref]: assigned1231lb # [ 0.424600] pci 0000:00:04.0: BAR 0 [io 0x1140-0x115f]: assigned1232worker1 # [ 0.413779] pci 0000:00:03.0: BAR 4 [mem 0x8000008000-0x800000bfff 64bit pref]: assigned1233lb # [ 0.424621] pci 0000:00:05.0: BAR 0 [io 0x1160-0x117f]: assigned1234client # [ 0.391222] pci 0000:00:0c.0: BAR 4 [mem 0x8000028000-0x800002bfff 64bit pref]: assigned1235worker1 # [ 0.413826] pci 0000:00:04.0: BAR 4 [mem 0x800000c000-0x800000ffff 64bit pref]: assigned1236lb # [ 0.424642] pci 0000:00:07.0: BAR 0 [io 0x1180-0x119f]: assigned1237client # [ 0.391267] pci 0000:00:01.0: BAR 1 [mem 0x10080000-0x10080fff]: assigned1238lb # [ 0.424664] pci 0000:00:0c.0: BAR 0 [io 0x11a0-0x11bf]: assigned1239worker1 # [ 0.413871] pci 0000:00:05.0: BAR 4 [mem 0x8000010000-0x8000013fff 64bit pref]: assigned1240client # [ 0.391289] pci 0000:00:02.0: BAR 1 [mem 0x10081000-0x10081fff]: assigned1241lb # [ 0.424690] pci_bus 0000:00: resource 4 [io 0x0000-0xffff]1242worker1 # [ 0.413918] pci 0000:00:06.0: BAR 4 [mem 0x8000014000-0x8000017fff 64bit pref]: assigned1243client # [ 0.391310] pci 0000:00:03.0: BAR 1 [mem 0x10082000-0x10082fff]: assigned1244lb # [ 0.424700] pci_bus 0000:00: resource 5 [mem 0x10000000-0x3efeffff]1245client # [ 0.391330] pci 0000:00:04.0: BAR 1 [mem 0x10083000-0x10083fff]: assigned1246worker1 # [ 0.413965] pci 0000:00:07.0: BAR 4 [mem 0x8000018000-0x800001bfff 64bit pref]: assigned1247lb # [ 0.424704] pci_bus 0000:00: resource 6 [mem 0x8000000000-0xffffffffff]1248client # [ 0.391353] pci 0000:00:05.0: BAR 1 [mem 0x10084000-0x10084fff]: assigned1249lb # [ 0.425846] pci 0000:00:0a.0: enabling device (0000 -> 0002)1250worker1 # [ 0.414012] pci 0000:00:08.0: BAR 4 [mem 0x800001c000-0x800001ffff 64bit pref]: assigned1251client # [ 0.391375] pci 0000:00:06.0: BAR 1 [mem 0x10085000-0x10085fff]: assigned1252worker1 # [ 0.414059] pci 0000:00:09.0: BAR 4 [mem 0x8000020000-0x8000023fff 64bit pref]: assigned1253client # [ 0.391397] pci 0000:00:07.0: BAR 1 [mem 0x10086000-0x10086fff]: assigned1254worker1 # [ 0.414105] pci 0000:00:0b.0: BAR 4 [mem 0x8000024000-0x8000027fff 64bit pref]: assigned1255client # [ 0.391419] pci 0000:00:08.0: BAR 1 [mem 0x10087000-0x10087fff]: assigned1256client # [ 0.391441] pci 0000:00:09.0: BAR 1 [mem 0x10088000-0x10088fff]: assigned1257worker1 # [ 0.414174] pci 0000:00:0c.0: BAR 4 [mem 0x8000028000-0x800002bfff 64bit pref]: assigned1258client # [ 0.391463] pci 0000:00:0a.0: BAR 0 [mem 0x10089000-0x10089fff]: assigned1259worker1 # [ 0.414228] pci 0000:00:01.0: BAR 1 [mem 0x10080000-0x10080fff]: assigned1260client # [ 0.391485] pci 0000:00:0b.0: BAR 1 [mem 0x1008a000-0x1008afff]: assigned1261worker1 # [ 0.414249] pci 0000:00:02.0: BAR 1 [mem 0x10081000-0x10081fff]: assigned1262client # [ 0.391507] pci 0000:00:0c.0: BAR 1 [mem 0x1008b000-0x1008bfff]: assigned1263worker1 # [ 0.414271] pci 0000:00:03.0: BAR 1 [mem 0x10082000-0x10082fff]: assigned1264client # [ 0.391528] pci 0000:00:06.0: BAR 0 [io 0x1000-0x107f]: assigned1265worker1 # [ 0.414292] pci 0000:00:04.0: BAR 1 [mem 0x10083000-0x10083fff]: assigned1266client # [ 0.391548] pci 0000:00:03.0: BAR 0 [io 0x1080-0x10bf]: assigned1267worker1 # [ 0.414317] pci 0000:00:05.0: BAR 1 [mem 0x10084000-0x10084fff]: assigned1268client # [ 0.391569] pci 0000:00:0b.0: BAR 0 [io 0x10c0-0x10ff]: assigned1269worker1 # [ 0.414338] pci 0000:00:06.0: BAR 1 [mem 0x10085000-0x10085fff]: assigned1270worker2 # [ 0.479326] virtio-pci 0000:00:01.0: enabling device (0000 -> 0003)1271client # [ 0.391590] pci 0000:00:01.0: BAR 0 [io 0x1100-0x111f]: assigned1272worker1 # [ 0.414360] pci 0000:00:07.0: BAR 1 [mem 0x10086000-0x10086fff]: assigned1273client # [ 0.391611] pci 0000:00:02.0: BAR 0 [io 0x1120-0x113f]: assigned1274worker1 # [ 0.414381] pci 0000:00:08.0: BAR 1 [mem 0x10087000-0x10087fff]: assigned1275client # [ 0.391632] pci 0000:00:04.0: BAR 0 [io 0x1140-0x115f]: assigned1276worker1 # [ 0.414403] pci 0000:00:09.0: BAR 1 [mem 0x10088000-0x10088fff]: assigned1277client # [ 0.391653] pci 0000:00:05.0: BAR 0 [io 0x1160-0x117f]: assigned1278worker1 # [ 0.414425] pci 0000:00:0a.0: BAR 0 [mem 0x10089000-0x10089fff]: assigned1279client # [ 0.391674] pci 0000:00:07.0: BAR 0 [io 0x1180-0x119f]: assigned1280worker1 # [ 0.414447] pci 0000:00:0b.0: BAR 1 [mem 0x1008a000-0x1008afff]: assigned1281client # [ 0.391696] pci 0000:00:0c.0: BAR 0 [io 0x11a0-0x11bf]: assigned1282client # [ 0.391721] pci_bus 0000:00: resource 4 [io 0x0000-0xffff]1283worker1 # [ 0.414469] pci 0000:00:0c.0: BAR 1 [mem 0x1008b000-0x1008bfff]: assigned1284client # [ 0.391731] pci_bus 0000:00: resource 5 [mem 0x10000000-0x3efeffff]1285worker1 # [ 0.414490] pci 0000:00:06.0: BAR 0 [io 0x1000-0x107f]: assigned1286client # [ 0.391735] pci_bus 0000:00: resource 6 [mem 0x8000000000-0xffffffffff]1287worker1 # [ 0.414511] pci 0000:00:03.0: BAR 0 [io 0x1080-0x10bf]: assigned1288worker1 # [ 0.414532] pci 0000:00:0b.0: BAR 0 [io 0x10c0-0x10ff]: assigned1289worker1 # [ 0.414553] pci 0000:00:01.0: BAR 0 [io 0x1100-0x111f]: assigned1290worker1 # [ 0.414575] pci 0000:00:02.0: BAR 0 [io 0x1120-0x113f]: assigned1291worker1 # [ 0.414596] pci 0000:00:04.0: BAR 0 [io 0x1140-0x115f]: assigned1292worker1 # [ 0.414617] pci 0000:00:05.0: BAR 0 [io 0x1160-0x117f]: assigned1293worker1 # [ 0.414639] pci 0000:00:07.0: BAR 0 [io 0x1180-0x119f]: assigned1294worker1 # [ 0.414660] pci 0000:00:0c.0: BAR 0 [io 0x11a0-0x11bf]: assigned1295worker1 # [ 0.414687] pci_bus 0000:00: resource 4 [io 0x0000-0xffff]1296worker1 # [ 0.414697] pci_bus 0000:00: resource 5 [mem 0x10000000-0x3efeffff]1297worker1 # [ 0.414701] pci_bus 0000:00: resource 6 [mem 0x8000000000-0xffffffffff]1298worker1 # [ 0.415845] pci 0000:00:0a.0: enabling device (0000 -> 0002)1299client # [ 0.456948] pci 0000:00:0a.0: enabling device (0000 -> 0002)1300lb # [ 0.481713] virtio-pci 0000:00:01.0: enabling device (0000 -> 0003)1301worker2 # [ 0.493741] virtio-pci 0000:00:02.0: enabling device (0000 -> 0003)1302worker2 # [ 0.495677] virtio-pci 0000:00:03.0: enabling device (0000 -> 0003)1303worker2 # [ 0.497768] virtio-pci 0000:00:04.0: enabling device (0000 -> 0003)1304worker2 # [ 0.499877] virtio-pci 0000:00:05.0: enabling device (0000 -> 0003)1305lb # [ 0.487744] virtio-pci 0000:00:02.0: enabling device (0000 -> 0003)1306lb # [ 0.489706] virtio-pci 0000:00:03.0: enabling device (0000 -> 0003)1307worker2 # [ 0.510338] virtio-pci 0000:00:06.0: enabling device (0000 -> 0003)1308worker2 # [ 0.512435] virtio-pci 0000:00:07.0: enabling device (0000 -> 0003)1309worker2 # [ 0.516076] virtio-pci 0000:00:08.0: enabling device (0000 -> 0002)1310lb # [ 0.499794] virtio-pci 0000:00:04.0: enabling device (0000 -> 0003)1311lb # [ 0.501942] virtio-pci 0000:00:05.0: enabling device (0000 -> 0003)1312client # [ 0.482047] virtio-pci 0000:00:01.0: enabling device (0000 -> 0003)1313worker1 # [ 0.491897] virtio-pci 0000:00:01.0: enabling device (0000 -> 0003)1314worker2 # [ 0.522864] virtio-pci 0000:00:09.0: enabling device (0000 -> 0002)1315client # [ 0.488252] virtio-pci 0000:00:02.0: enabling device (0000 -> 0003)1316client # [ 0.490352] virtio-pci 0000:00:03.0: enabling device (0000 -> 0003)1317worker1 # [ 0.498259] virtio-pci 0000:00:02.0: enabling device (0000 -> 0003)1318worker1 # [ 0.500175] virtio-pci 0000:00:03.0: enabling device (0000 -> 0003)1319lb # [ 0.511694] virtio-pci 0000:00:06.0: enabling device (0000 -> 0003)1320lb # [ 0.514433] virtio-pci 0000:00:07.0: enabling device (0000 -> 0003)1321lb # [ 0.516565] virtio-pci 0000:00:08.0: enabling device (0000 -> 0002)1322lb # [ 0.518648] virtio-pci 0000:00:09.0: enabling device (0000 -> 0002)1323client # [ 0.500219] virtio-pci 0000:00:04.0: enabling device (0000 -> 0003)1324client # [ 0.502198] virtio-pci 0000:00:05.0: enabling device (0000 -> 0003)1325worker2 # [ 0.537624] virtio-pci 0000:00:0b.0: enabling device (0000 -> 0003)1326worker2 # [ 0.539887] virtio-pci 0000:00:0c.0: enabling device (0000 -> 0003)1327worker1 # [ 0.510642] virtio-pci 0000:00:04.0: enabling device (0000 -> 0003)1328worker1 # [ 0.512652] virtio-pci 0000:00:05.0: enabling device (0000 -> 0003)1329worker2 # [ 0.546345] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled1330lb # [ 0.528590] virtio-pci 0000:00:0b.0: enabling device (0000 -> 0003)1331lb # [ 0.530893] virtio-pci 0000:00:0c.0: enabling device (0000 -> 0003)1332worker2 # [ 0.550252] msm_serial: driver initialized1333worker2 # [ 0.550446] SuperH (H)SCI(F) driver initialized1334worker2 # [ 0.550499] STM32 USART driver initialized1335worker1 # [ 0.523889] virtio-pci 0000:00:06.0: enabling device (0000 -> 0003)1336client # [ 0.512195] virtio-pci 0000:00:06.0: enabling device (0000 -> 0003)1337client # [ 0.514212] virtio-pci 0000:00:07.0: enabling device (0000 -> 0003)1338client # [ 0.516350] virtio-pci 0000:00:08.0: enabling device (0000 -> 0002)1339worker1 # [ 0.527032] virtio-pci 0000:00:07.0: enabling device (0000 -> 0003)1340client # [ 0.518399] virtio-pci 0000:00:09.0: enabling device (0000 -> 0002)1341worker1 # [ 0.529040] virtio-pci 0000:00:08.0: enabling device (0000 -> 0002)1342client # [ 0.528192] virtio-pci 0000:00:0b.0: enabling device (0000 -> 0003)1343client # [ 0.530387] virtio-pci 0000:00:0c.0: enabling device (0000 -> 0003)1344worker1 # [ 0.539015] virtio-pci 0000:00:09.0: enabling device (0000 -> 0002)1345worker1 # [ 0.540873] virtio-pci 0000:00:0b.0: enabling device (0000 -> 0003)1346lb # [ 0.556402] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled1347lb # [ 0.558139] msm_serial: driver initialized1348lb # [ 0.558277] SuperH (H)SCI(F) driver initialized1349lb # [ 0.558330] STM32 USART driver initialized1350client # [ 0.543081] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled1351worker1 # [ 0.551514] virtio-pci 0000:00:0c.0: enabling device (0000 -> 0003)1352worker2 # [ 0.583780] loop: module loaded1353client # [ 0.545646] msm_serial: driver initialized1354worker2 # [ 0.583958] virtio_blk virtio5: 1/0/0 default/read/poll queues1355client # [ 0.545835] SuperH (H)SCI(F) driver initialized1356worker2 # [ 0.584688] virtio_blk virtio5: [vda] 2097152 512-byte logical blocks (1.07 GB/1.00 GiB)1357client # [ 0.545887] STM32 USART driver initialized1358worker2 # [ 0.589681] megasas: 07.734.00.00-rc11359worker2 # [ 0.590391] physmap-flash 0.flash: physmap platform flash device: [mem 0x00000000-0x03ffffff]1360worker1 # [ 0.562439] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled1361worker1 # [ 0.565026] msm_serial: driver initialized1362worker1 # [ 0.565175] SuperH (H)SCI(F) driver initialized1363worker1 # [ 0.566369] STM32 USART driver initialized1364lb # [ 0.587735] loop: module loaded1365lb # [ 0.587967] virtio_blk virtio5: 1/0/0 default/read/poll queues1366lb # [ 0.588690] virtio_blk virtio5: [vda] 2097152 512-byte logical blocks (1.07 GB/1.00 GiB)1367worker2 # [ 0.601119] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001368worker2 # [ 0.601142] Intel/Sharp Extended Query Table at 0x00311369worker2 # [ 0.602899] Using buffer write method1370worker2 # [ 0.602961] physmap-flash 0.flash: physmap platform flash device: [mem 0x04000000-0x07ffffff]1371worker2 # [ 0.604584] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001372worker2 # [ 0.604633] Intel/Sharp Extended Query Table at 0x00311373worker2 # [ 0.606291] Using buffer write method1374worker2 # [ 0.606310] Concatenating MTD devices:1375worker2 # [ 0.606314] (0): "0.flash"1376worker2 # [ 0.606318] (1): "0.flash"1377worker2 # [ 0.606322] into device "0.flash"1378lb # [ 0.595759] megasas: 07.734.00.00-rc11379client # [ 0.580662] loop: module loaded1380lb # [ 0.596511] physmap-flash 0.flash: physmap platform flash device: [mem 0x00000000-0x03ffffff]1381client # [ 0.580845] virtio_blk virtio5: 1/0/0 default/read/poll queues1382lb # [ 0.598693] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001383client # [ 0.581543] virtio_blk virtio5: [vda] 2097152 512-byte logical blocks (1.07 GB/1.00 GiB)1384lb # [ 0.598716] Intel/Sharp Extended Query Table at 0x00311385lb # [ 0.600286] Using buffer write method1386lb # [ 0.600350] physmap-flash 0.flash: physmap platform flash device: [mem 0x04000000-0x07ffffff]1387lb # [ 0.601976] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001388lb # [ 0.602002] Intel/Sharp Extended Query Table at 0x00311389worker1 # [ 0.598127] loop: module loaded1390worker1 # [ 0.598307] virtio_blk virtio5: 1/0/0 default/read/poll queues1391lb # [ 0.615716] Using buffer write method1392worker1 # [ 0.599037] virtio_blk virtio5: [vda] 2097152 512-byte logical blocks (1.07 GB/1.00 GiB)1393lb # [ 0.615743] Concatenating MTD devices:1394lb # [ 0.615747] (0): "0.flash"1395lb # [ 0.615751] (1): "0.flash"1396lb # [ 0.615755] into device "0.flash"1397client # [ 0.588644] megasas: 07.734.00.00-rc11398client # [ 0.589310] physmap-flash 0.flash: physmap platform flash device: [mem 0x00000000-0x03ffffff]1399client # [ 0.591693] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001400client # [ 0.591719] Intel/Sharp Extended Query Table at 0x00311401client # [ 0.593257] Using buffer write method1402client # [ 0.593315] physmap-flash 0.flash: physmap platform flash device: [mem 0x04000000-0x07ffffff]1403client # [ 0.594957] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001404client # [ 0.594998] Intel/Sharp Extended Query Table at 0x00311405worker1 # [ 0.609790] megasas: 07.734.00.00-rc11406worker1 # [ 0.610483] physmap-flash 0.flash: physmap platform flash device: [mem 0x00000000-0x03ffffff]1407worker1 # [ 0.612597] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001408worker1 # [ 0.612622] Intel/Sharp Extended Query Table at 0x00311409worker1 # [ 0.614192] Using buffer write method1410client # [ 0.612597] Using buffer write method1411client # [ 0.612630] Concatenating MTD devices:1412client # [ 0.612634] (0): "0.flash"1413worker1 # [ 0.614257] physmap-flash 0.flash: physmap platform flash device: [mem 0x04000000-0x07ffffff]1414client # [ 0.612638] (1): "0.flash"1415client # [ 0.612641] into device "0.flash"1416worker1 # [ 0.615858] 0.flash: Found 2 x16 devices at 0x0 in 32-bit bank. Manufacturer ID 0x000000 Chip ID 0x0000001417worker1 # [ 0.615884] Intel/Sharp Extended Query Table at 0x00311418worker1 # [ 0.634707] Using buffer write method1419worker1 # [ 0.634744] Concatenating MTD devices:1420worker1 # [ 0.634748] (0): "0.flash"1421worker1 # [ 0.634753] (1): "0.flash"1422worker1 # [ 0.634756] into device "0.flash"1423worker2 # [ 0.841801] Freeing initrd memory: 26148K1424worker2 # [ 0.847768] tun: Universal TUN/TAP device driver, 1.61425worker2 # [ 0.851419] thunder_xcv, ver 1.01426worker2 # [ 0.851463] thunder_bgx, ver 1.01427worker2 # [ 0.851487] nicpf, ver 1.01428worker2 # [ 0.852062] e1000: Intel(R) PRO/1000 Network Driver1429worker2 # [ 0.852070] e1000: Copyright (c) 1999-2006 Intel Corporation.1430worker2 # [ 0.852093] e1000e: Intel(R) PRO/1000 Network Driver1431worker2 # [ 0.852102] e1000e: Copyright(c) 1999 - 2015 Intel Corporation.1432worker2 # [ 0.852132] igb: Intel(R) Gigabit Ethernet Network Driver1433worker2 # [ 0.852137] igb: Copyright (c) 2007-2014 Intel Corporation.1434worker2 # [ 0.852160] igbvf: Intel(R) Gigabit Virtual Function Network Driver1435worker2 # [ 0.852166] igbvf: Copyright (c) 2009 - 2012 Intel Corporation.1436worker2 # [ 0.852303] sky2: driver version 1.301437worker2 # [ 0.860818] usbcore: registered new interface driver usb-storage1438worker2 # [ 0.860955] usbcore: registered new interface driver usbserial_generic1439worker2 # [ 0.860971] usbserial: USB Serial support registered for generic1440worker2 # [ 0.861762] ehci-pci 0000:00:0a.0: EHCI Host Controller1441worker2 # [ 0.861785] ehci-pci 0000:00:0a.0: new USB bus registered, assigned bus number 11442worker2 # [ 0.862019] ehci-pci 0000:00:0a.0: irq 15, io mem 0x100890001443worker2 # [ 0.867076] hv_vmbus: registering driver hyperv_keyboard1444lb # [ 0.850524] Freeing initrd memory: 26140K1445worker2 # [ 0.868586] rtc-pl031 9010000.pl031: registered as rtc01446worker2 # [ 0.868613] rtc-pl031 9010000.pl031: setting system clock to 2026-09-22T10:35:57 UTC (1790073357)1447worker2 # [ 0.868911] i2c_dev: i2c /dev entries driver1448lb # [ 0.856635] tun: Universal TUN/TAP device driver, 1.61449worker2 # [ 0.873905] sdhci: Secure Digital Host Controller Interface driver1450worker2 # [ 0.873915] sdhci: Copyright(c) Pierre Ossman1451worker2 # [ 0.874181] Synopsys Designware Multimedia Card Interface Driver1452worker2 # [ 0.874558] sdhci-pltfm: SDHCI platform and OF driver helper1453worker2 # [ 0.876278] hid: raw HID events driver (C) Jiri Kosina1454lb # [ 0.860600] thunder_xcv, ver 1.01455lb # [ 0.860645] thunder_bgx, ver 1.01456worker2 # [ 0.876540] usbcore: registered new interface driver usbhid1457lb # [ 0.860668] nicpf, ver 1.01458worker2 # [ 0.876547] usbhid: USB HID core driver1459lb # [ 0.861222] e1000: Intel(R) PRO/1000 Network Driver1460lb # [ 0.861229] e1000: Copyright (c) 1999-2006 Intel Corporation.1461worker2 # [ 0.881579] ehci-pci 0000:00:0a.0: USB 2.0 started, EHCI 1.001462lb # [ 0.861253] e1000e: Intel(R) PRO/1000 Network Driver1463worker2 # [ 0.881910] hub 1-0:1.0: USB hub found1464lb # [ 0.861261] e1000e: Copyright(c) 1999 - 2015 Intel Corporation.1465worker2 # [ 0.881930] hub 1-0:1.0: 6 ports detected1466lb # [ 0.861289] igb: Intel(R) Gigabit Ethernet Network Driver1467lb # [ 0.861295] igb: Copyright (c) 2007-2014 Intel Corporation.1468lb # [ 0.861319] igbvf: Intel(R) Gigabit Virtual Function Network Driver1469client # [ 0.846178] Freeing initrd memory: 26160K1470worker2 # [ 0.884521] hw perfevents: enabled with armv8_pmuv3 PMU driver, 11 (0,800003ff) counters available1471lb # [ 0.861324] igbvf: Copyright (c) 2009 - 2012 Intel Corporation.1472lb # [ 0.861465] sky2: driver version 1.301473lb # [ 0.863082] usbcore: registered new interface driver usb-storage1474worker2 # [ 0.887128] drop_monitor: Initializing network drop monitor service1475lb # [ 0.863906] ehci-pci 0000:00:0a.0: EHCI Host Controller1476worker2 # [ 0.887318] NET: Registered PF_INET6 protocol family1477lb # [ 0.863932] ehci-pci 0000:00:0a.0: new USB bus registered, assigned bus number 11478worker2 # [ 0.889289] Segment Routing with IPv61479lb # [ 0.864207] ehci-pci 0000:00:0a.0: irq 15, io mem 0x100890001480worker2 # [ 0.889305] In-situ OAM (IOAM) with IPv61481worker1 # [ 0.859019] Freeing initrd memory: 26148K1482client # [ 0.852034] tun: Universal TUN/TAP device driver, 1.61483worker2 # [ 0.889356] NET: Registered PF_PACKET protocol family1484lb # [ 0.874122] usbcore: registered new interface driver usbserial_generic1485lb # [ 0.874143] usbserial: USB Serial support registered for generic1486lb # [ 0.874803] hv_vmbus: registering driver hyperv_keyboard1487worker2 # [ 0.890946] 9pnet: Installing 9P2000 support1488lb # [ 0.877357] ehci-pci 0000:00:0a.0: USB 2.0 started, EHCI 1.001489worker2 # [ 0.893062] Key type dns_resolver registered1490client # [ 0.855796] thunder_xcv, ver 1.01491client # [ 0.855840] thunder_bgx, ver 1.01492lb # [ 0.878375] hub 1-0:1.0: USB hub found1493client # [ 0.855863] nicpf, ver 1.01494lb # [ 0.878857] hub 1-0:1.0: 6 ports detected1495worker1 # [ 0.865058] tun: Universal TUN/TAP device driver, 1.61496lb # [ 0.880232] rtc-pl031 9010000.pl031: registered as rtc01497lb # [ 0.880258] rtc-pl031 9010000.pl031: setting system clock to 2026-09-22T10:35:57 UTC (1790073357)1498lb # [ 0.880561] i2c_dev: i2c /dev entries driver1499client # [ 0.857645] e1000: Intel(R) PRO/1000 Network Driver1500client # [ 0.857656] e1000: Copyright (c) 1999-2006 Intel Corporation.1501worker1 # [ 0.868790] thunder_xcv, ver 1.01502client # [ 0.857681] e1000e: Intel(R) PRO/1000 Network Driver1503worker1 # [ 0.868845] thunder_bgx, ver 1.01504worker1 # [ 0.868867] nicpf, ver 1.01505worker2 # [ 0.899886] registered taskstats version 11506client # [ 0.857691] e1000e: Copyright(c) 1999 - 2015 Intel Corporation.1507worker2 # [ 0.900051] Loading compiled-in X.509 certificates1508client # [ 0.857720] igb: Intel(R) Gigabit Ethernet Network Driver1509client # [ 0.857726] igb: Copyright (c) 2007-2014 Intel Corporation.1510client # [ 0.857749] igbvf: Intel(R) Gigabit Virtual Function Network Driver1511lb # [ 0.885428] sdhci: Secure Digital Host Controller Interface driver1512lb # [ 0.885439] sdhci: Copyright(c) Pierre Ossman1513client # [ 0.857755] igbvf: Copyright (c) 2009 - 2012 Intel Corporation.1514worker1 # [ 0.870649] e1000: Intel(R) PRO/1000 Network Driver1515client # [ 0.857892] sky2: driver version 1.301516lb # [ 0.885704] Synopsys Designware Multimedia Card Interface Driver1517worker1 # [ 0.870660] e1000: Copyright (c) 1999-2006 Intel Corporation.1518client # [ 0.859500] usbcore: registered new interface driver usb-storage1519lb # [ 0.886072] sdhci-pltfm: SDHCI platform and OF driver helper1520worker1 # [ 0.870684] e1000e: Intel(R) PRO/1000 Network Driver1521client # [ 0.859591] usbcore: registered new interface driver usbserial_generic1522worker1 # [ 0.870693] e1000e: Copyright(c) 1999 - 2015 Intel Corporation.1523client # [ 0.859605] usbserial: USB Serial support registered for generic1524worker1 # [ 0.870724] igb: Intel(R) Gigabit Ethernet Network Driver1525lb # [ 0.890377] hid: raw HID events driver (C) Jiri Kosina1526client # [ 0.860748] ehci-pci 0000:00:0a.0: EHCI Host Controller1527worker1 # [ 0.870729] igb: Copyright (c) 2007-2014 Intel Corporation.1528lb # [ 0.890644] usbcore: registered new interface driver usbhid1529client # [ 0.860773] ehci-pci 0000:00:0a.0: new USB bus registered, assigned bus number 11530lb # [ 0.890650] usbhid: USB HID core driver1531worker1 # [ 0.870753] igbvf: Intel(R) Gigabit Virtual Function Network Driver1532client # [ 0.861082] ehci-pci 0000:00:0a.0: irq 15, io mem 0x100890001533worker2 # [ 0.908580] Demotion targets for Node 0: null1534worker2 # [ 0.908685] Key type .fscrypt registered1535worker1 # [ 0.870759] igbvf: Copyright (c) 2009 - 2012 Intel Corporation.1536worker2 # [ 0.908691] Key type fscrypt-provisioning registered1537worker1 # [ 0.870900] sky2: driver version 1.301538client # [ 0.871641] hv_vmbus: registering driver hyperv_keyboard1539worker2 # [ 0.908784] ima: No TPM chip found, activating TPM-bypass!1540worker1 # [ 0.872483] usbcore: registered new interface driver usb-storage1541client # [ 0.872082] ehci-pci 0000:00:0a.0: USB 2.0 started, EHCI 1.001542worker2 # [ 0.908803] ima: Allocated hash algorithm: sha11543lb # [ 0.893456] hw perfevents: enabled with armv8_pmuv3 PMU driver, 11 (0,800003ff) counters available1544client # [ 0.872410] hub 1-0:1.0: USB hub found1545worker2 # [ 0.908823] ima: No architecture policies found1546worker1 # [ 0.872535] usbcore: registered new interface driver usbserial_generic1547client # [ 0.872430] hub 1-0:1.0: 6 ports detected1548lb # [ 0.894955] drop_monitor: Initializing network drop monitor service1549worker1 # [ 0.872550] usbserial: USB Serial support registered for generic1550lb # [ 0.895084] NET: Registered PF_INET6 protocol family1551worker1 # [ 0.878554] ehci-pci 0000:00:0a.0: EHCI Host Controller1552worker2 # [ 0.912905] input: gpio-keys as /devices/platform/gpio-keys/input/input01553lb # [ 0.898020] Segment Routing with IPv61554client # [ 0.875388] rtc-pl031 9010000.pl031: registered as rtc01555worker1 # [ 0.878573] ehci-pci 0000:00:0a.0: new USB bus registered, assigned bus number 11556lb # [ 0.898052] In-situ OAM (IOAM) with IPv61557worker1 # [ 0.878766] ehci-pci 0000:00:0a.0: irq 15, io mem 0x100890001558client # [ 0.875417] rtc-pl031 9010000.pl031: setting system clock to 2026-09-22T10:35:57 UTC (1790073357)1559lb # [ 0.898081] NET: Registered PF_PACKET protocol family1560client # [ 0.875721] i2c_dev: i2c /dev entries driver1561worker1 # [ 0.884494] hv_vmbus: registering driver hyperv_keyboard1562worker1 # [ 0.886079] rtc-pl031 9010000.pl031: registered as rtc01563lb # [ 0.899683] 9pnet: Installing 9P2000 support1564worker1 # [ 0.886106] rtc-pl031 9010000.pl031: setting system clock to 2026-09-22T10:35:57 UTC (1790073357)1565lb # [ 0.901713] Key type dns_resolver registered1566worker1 # [ 0.886408] i2c_dev: i2c /dev entries driver1567worker1 # [ 0.889525] ehci-pci 0000:00:0a.0: USB 2.0 started, EHCI 1.001568worker1 # [ 0.890532] hub 1-0:1.0: USB hub found1569worker1 # [ 0.891015] hub 1-0:1.0: 6 ports detected1570client # [ 0.880573] sdhci: Secure Digital Host Controller Interface driver1571client # [ 0.880583] sdhci: Copyright(c) Pierre Ossman1572client # [ 0.880844] Synopsys Designware Multimedia Card Interface Driver1573client # [ 0.881205] sdhci-pltfm: SDHCI platform and OF driver helper1574client # [ 0.882940] hid: raw HID events driver (C) Jiri Kosina1575client # [ 0.883190] usbcore: registered new interface driver usbhid1576client # [ 0.883196] usbhid: USB HID core driver1577lb # [ 0.908757] registered taskstats version 11578lb # [ 0.908913] Loading compiled-in X.509 certificates1579worker1 # [ 0.894012] sdhci: Secure Digital Host Controller Interface driver1580worker1 # [ 0.894022] sdhci: Copyright(c) Pierre Ossman1581client # [ 0.888694] hw perfevents: enabled with armv8_pmuv3 PMU driver, 11 (0,800003ff) counters available1582worker1 # [ 0.894293] Synopsys Designware Multimedia Card Interface Driver1583client # [ 0.890193] drop_monitor: Initializing network drop monitor service1584client # [ 0.890369] NET: Registered PF_INET6 protocol family1585worker1 # [ 0.894664] sdhci-pltfm: SDHCI platform and OF driver helper1586worker1 # [ 0.896379] hid: raw HID events driver (C) Jiri Kosina1587client # [ 0.893407] Segment Routing with IPv61588worker1 # [ 0.896630] usbcore: registered new interface driver usbhid1589client # [ 0.893425] In-situ OAM (IOAM) with IPv61590worker1 # [ 0.896636] usbhid: USB HID core driver1591client # [ 0.893454] NET: Registered PF_PACKET protocol family1592worker2 # [ 0.930423] clk: Disabling unused clocks1593worker2 # [ 0.930451] PM: genpd: Disabling unused power domains1594lb # [ 0.917418] Demotion targets for Node 0: null1595client # [ 0.895096] 9pnet: Installing 9P2000 support1596lb # [ 0.917527] Key type .fscrypt registered1597lb # [ 0.917533] Key type fscrypt-provisioning registered1598worker2 # [ 0.934657] Freeing unused kernel memory: 4736K1599worker1 # [ 0.902248] hw perfevents: enabled with armv8_pmuv3 PMU driver, 11 (0,800003ff) counters available1600lb # [ 0.917627] ima: No TPM chip found, activating TPM-bypass!1601worker2 # [ 0.934843] Run /init as init process1602lb # [ 0.917647] ima: Allocated hash algorithm: sha11603worker1 # [ 0.903781] drop_monitor: Initializing network drop monitor service1604lb # [ 0.917669] ima: No architecture policies found1605worker1 # [ 0.903927] NET: Registered PF_INET6 protocol family1606client # [ 0.897914] Key type dns_resolver registered1607lb # [ 0.921713] input: gpio-keys as /devices/platform/gpio-keys/input/input01608worker1 # [ 0.906998] Segment Routing with IPv61609worker1 # [ 0.907028] In-situ OAM (IOAM) with IPv61610worker1 # [ 0.907058] NET: Registered PF_PACKET protocol family1611worker1 # [ 0.908665] 9pnet: Installing 9P2000 support1612worker1 # [ 0.911268] Key type dns_resolver registered1613client # [ 0.904182] registered taskstats version 11614client # [ 0.904341] Loading compiled-in X.509 certificates1615client # [ 0.912806] Demotion targets for Node 0: null1616worker2 # [ 0.951294] systemd[1]: Successfully made /usr/ read-only.1617client # [ 0.912909] Key type .fscrypt registered1618client # [ 0.912915] Key type fscrypt-provisioning registered1619worker1 # [ 0.917911] registered taskstats version 11620worker1 # [ 0.918072] Loading compiled-in X.509 certificates1621client # [ 0.913013] ima: No TPM chip found, activating TPM-bypass!1622client # [ 0.913032] ima: Allocated hash algorithm: sha11623client # [ 0.913053] ima: No architecture policies found1624client # [ 0.917093] input: gpio-keys as /devices/platform/gpio-keys/input/input01625lb # [ 0.939301] clk: Disabling unused clocks1626lb # [ 0.939322] PM: genpd: Disabling unused power domains1627worker1 # [ 0.926540] Demotion targets for Node 0: null1628worker1 # [ 0.926645] Key type .fscrypt registered1629worker1 # [ 0.926651] Key type fscrypt-provisioning registered1630worker1 # [ 0.926743] ima: No TPM chip found, activating TPM-bypass!1631lb # [ 0.943515] Freeing unused kernel memory: 4736K1632worker1 # [ 0.926762] ima: Allocated hash algorithm: sha11633lb # [ 0.943716] Run /init as init process1634worker1 # [ 0.926784] ima: No architecture policies found1635worker1 # [ 0.930854] input: gpio-keys as /devices/platform/gpio-keys/input/input01636client # [ 0.935025] clk: Disabling unused clocks1637client # [ 0.935058] PM: genpd: Disabling unused power domains1638lb # [ 0.961464] systemd[1]: Successfully made /usr/ read-only.1639client # [ 0.939302] Freeing unused kernel memory: 4736K1640client # [ 0.939488] Run /init as init process1641worker1 # [ 0.949351] clk: Disabling unused clocks1642worker1 # [ 0.949378] PM: genpd: Disabling unused power domains1643worker1 # [ 0.953615] Freeing unused kernel memory: 4736K1644worker1 # [ 0.953800] Run /init as init process1645client # [ 0.954309] systemd[1]: Successfully made /usr/ read-only.1646worker1 # [ 0.971358] systemd[1]: Successfully made /usr/ read-only.1647worker2 # [ 1.129123] usb 1-1: new high-speed USB device number 2 using ehci-pci1648lb # [ 1.127247] usb 1-1: new high-speed USB device number 2 using ehci-pci1649client # [ 1.120083] usb 1-1: new high-speed USB device number 2 using ehci-pci1650worker1 # [ 1.137260] usb 1-1: new high-speed USB device number 2 using ehci-pci1651worker2 # [ 1.281340] input: QEMU QEMU USB Keyboard as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-1/1-1:1.0/0003:0627:0001.0001/input/input11652lb # [ 1.279276] input: QEMU QEMU USB Keyboard as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-1/1-1:1.0/0003:0627:0001.0001/input/input11653worker2 # [ 1.287167] systemd[1]: systemd 261.2 running in system mode (+PAM +AUDIT -SELINUX +APPARMOR +IMA +IPE +SMACK +SECCOMP +GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 +KMOD +LIBCRYPTSETUP +LIBCRYPTSETUP_PLUGINS +LIBFDISK +PCRE2 +PWQUALITY +P11KIT +QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK -BTF -XKBCOMMON +UTMP +LIBARCHIVE)1654worker2 # [ 1.299365] systemd[1]: Detected virtualization qemu.1655worker2 # [ 1.299454] systemd[1]: Detected architecture arm64.1656worker2 # [ 1.299479] systemd[1]: Running in initrd.1657worker2 # [ 1.300418] systemd[1]: Initializing machine ID from random generator.1658worker2 # [ 1.300693] systemd[1]: Hostname set to <worker2>.1659client # [ 1.274542] input: QEMU QEMU USB Keyboard as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-1/1-1:1.0/0003:0627:0001.0001/input/input11660lb # [ 1.296339] systemd[1]: systemd 261.2 running in system mode (+PAM +AUDIT -SELINUX +APPARMOR +IMA +IPE +SMACK +SECCOMP +GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 +KMOD +LIBCRYPTSETUP +LIBCRYPTSETUP_PLUGINS +LIBFDISK +PCRE2 +PWQUALITY +P11KIT +QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK -BTF -XKBCOMMON +UTMP +LIBARCHIVE)1661lb # [ 1.308752] systemd[1]: Detected virtualization qemu.1662worker1 # [ 1.291727] input: QEMU QEMU USB Keyboard as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-1/1-1:1.0/0003:0627:0001.0001/input/input11663lb # [ 1.310756] systemd[1]: Detected architecture arm64.1664lb # [ 1.312764] systemd[1]: Running in initrd.1665lb # [ 1.315438] systemd[1]: Initializing machine ID from random generator.1666lb # [ 1.318325] systemd[1]: Hostname set to <lb>.1667client # [ 1.289181] systemd[1]: systemd 261.2 running in system mode (+PAM +AUDIT -SELINUX +APPARMOR +IMA +IPE +SMACK +SECCOMP +GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 +KMOD +LIBCRYPTSETUP +LIBCRYPTSETUP_PLUGINS +LIBFDISK +PCRE2 +PWQUALITY +P11KIT +QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK -BTF -XKBCOMMON +UTMP +LIBARCHIVE)1668client # [ 1.301432] systemd[1]: Detected virtualization qemu.1669client # [ 1.303545] systemd[1]: Detected architecture arm64.1670client # [ 1.305584] systemd[1]: Running in initrd.1671client # [ 1.308239] systemd[1]: Initializing machine ID from random generator.1672worker1 # [ 1.306201] systemd[1]: systemd 261.2 running in system mode (+PAM +AUDIT -SELINUX +APPARMOR +IMA +IPE +SMACK +SECCOMP +GCRYPT -GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 +KMOD +LIBCRYPTSETUP +LIBCRYPTSETUP_PLUGINS +LIBFDISK +PCRE2 +PWQUALITY +P11KIT +QRENCODE +TPM2 +BZIP2 +LZ4 +XZ +ZLIB +ZSTD +BPF_FRAMEWORK -BTF -XKBCOMMON +UTMP +LIBARCHIVE)1673client # [ 1.311092] systemd[1]: Hostname set to <client>.1674worker1 # [ 1.318459] systemd[1]: Detected virtualization qemu.1675worker1 # [ 1.320547] systemd[1]: Detected architecture arm64.1676worker1 # [ 1.322636] systemd[1]: Running in initrd.1677worker1 # [ 1.325448] systemd[1]: Initializing machine ID from random generator.1678worker1 # [ 1.328302] systemd[1]: Hostname set to <worker1>.1679worker2 # [ 1.365493] hid-generic 0003:0627:0001.0001: input,hidraw0: USB HID v1.11 Keyboard [QEMU QEMU USB Keyboard] on usb-0000:00:0a.0-1/input01680lb # [ 1.371549] hid-generic 0003:0627:0001.0001: input,hidraw0: USB HID v1.11 Keyboard [QEMU QEMU USB Keyboard] on usb-0000:00:0a.0-1/input01681client # [ 1.356477] hid-generic 0003:0627:0001.0001: input,hidraw0: USB HID v1.11 Keyboard [QEMU QEMU USB Keyboard] on usb-0000:00:0a.0-1/input01682worker1 # [ 1.373581] hid-generic 0003:0627:0001.0001: input,hidraw0: USB HID v1.11 Keyboard [QEMU QEMU USB Keyboard] on usb-0000:00:0a.0-1/input01683worker2 # [ 1.485137] usb 1-2: new high-speed USB device number 3 using ehci-pci1684lb # [ 1.491206] usb 1-2: new high-speed USB device number 3 using ehci-pci1685client # [ 1.480114] usb 1-2: new high-speed USB device number 3 using ehci-pci1686worker1 # [ 1.493250] usb 1-2: new high-speed USB device number 3 using ehci-pci1687worker2 # [ 1.645782] input: QEMU QEMU USB Tablet as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-2/1-2:1.0/0003:0627:0001.0002/input/input21688worker2 # [ 1.651626] hid-generic 0003:0627:0001.0002: input,hidraw1: USB HID v0.01 Mouse [QEMU QEMU USB Tablet] on usb-0000:00:0a.0-2/input01689lb # [ 1.645885] systemd[1]: bpf-restrict-fs: LSM BPF program attached1690worker1 # [ 1.634337] systemd[1]: bpf-restrict-fs: LSM BPF program attached1691worker2 # [ 1.664151] systemd[1]: bpf-restrict-fs: LSM BPF program attached1692client # [ 1.633470] systemd[1]: bpf-restrict-fs: LSM BPF program attached1693client # [ 1.636199] input: QEMU QEMU USB Tablet as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-2/1-2:1.0/0003:0627:0001.0002/input/input21694client # [ 1.638781] hid-generic 0003:0627:0001.0002: input,hidraw1: USB HID v0.01 Mouse [QEMU QEMU USB Tablet] on usb-0000:00:0a.0-2/input01695lb # [ 1.663274] input: QEMU QEMU USB Tablet as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-2/1-2:1.0/0003:0627:0001.0002/input/input21696lb # [ 1.668639] hid-generic 0003:0627:0001.0002: input,hidraw1: USB HID v0.01 Mouse [QEMU QEMU USB Tablet] on usb-0000:00:0a.0-2/input01697worker1 # [ 1.656323] input: QEMU QEMU USB Tablet as /devices/platform/4010000000.pcie/pci0000:00/0000:00:0a.0/usb1/1-2/1-2:1.0/0003:0627:0001.0002/input/input21698worker1 # [ 1.658724] hid-generic 0003:0627:0001.0002: input,hidraw1: USB HID v0.01 Mouse [QEMU QEMU USB Tablet] on usb-0000:00:0a.0-2/input01699worker1 # [ 1.708629] systemd[1]: Queued start job for default target Initrd Default Target.1700client # [ 1.702460] systemd[1]: Queued start job for default target Initrd Default Target.1701client # [ 1.710059] systemd[1]: Created slice Slice /system/modprobe.1702client # [ 1.711033] systemd[1]: Started Dispatch Password Requests to Console Directory Watch.1703client # [ 1.712148] systemd[1]: Expecting device /dev/disk/by-label/nixos...1704client # [ 1.712972] systemd[1]: Reached target Path Units.1705worker1 # [ 1.719597] systemd[1]: Created slice Slice /system/modprobe.1706client # [ 1.713618] systemd[1]: Reached target Slice Units.1707client # [ 1.714284] systemd[1]: Reached target Swaps.1708worker1 # [ 1.720780] systemd[1]: Started Dispatch Password Requests to Console Directory Watch.1709client # [ 1.714878] systemd[1]: Reached target Timer Units.1710worker1 # [ 1.721967] systemd[1]: Expecting device /dev/disk/by-label/nixos...1711worker1 # [ 1.722802] systemd[1]: Reached target Path Units.1712client # [ 1.715689] systemd[1]: Listening on D-Bus System Message Bus Socket.1713worker1 # [ 1.723451] systemd[1]: Reached target Slice Units.1714client # [ 1.716845] systemd[1]: Listening on Journal Socket (/dev/log).1715worker1 # [ 1.724123] systemd[1]: Reached target Swaps.1716client # [ 1.717744] systemd[1]: Listening on Journal Sockets.1717worker1 # [ 1.724723] systemd[1]: Reached target Timer Units.1718client # [ 1.718525] systemd[1]: Listening on udev Control Socket.1719worker1 # [ 1.725581] systemd[1]: Listening on D-Bus System Message Bus Socket.1720client # [ 1.719345] systemd[1]: Listening on udev Kernel Socket.1721worker1 # [ 1.726635] systemd[1]: Listening on Journal Socket (/dev/log).1722client # [ 1.720075] systemd[1]: Reached target Socket Units.1723worker1 # [ 1.727534] systemd[1]: Listening on Journal Sockets.1724worker1 # [ 1.728336] systemd[1]: Listening on udev Control Socket.1725worker1 # [ 1.729159] systemd[1]: Listening on udev Kernel Socket.1726client # [ 1.722203] systemd[1]: Starting Create List of Static Device Nodes...1727worker1 # [ 1.729982] systemd[1]: Reached target Socket Units.1728worker1 # [ 1.732070] systemd[1]: Starting Create List of Static Device Nodes...1729worker1 # [ 1.738307] systemd[1]: Starting Load Kernel Module 9pnet_virtio...1730client # [ 1.731961] systemd[1]: Starting Load Kernel Module 9pnet_virtio...1731worker1 # [ 1.739245] systemd[1]: Load Kernel Module configfs skipped, unmet condition check ConditionKernelModuleLoaded=!configfs1732client # [ 1.733017] systemd[1]: Load Kernel Module configfs skipped, unmet condition check ConditionKernelModuleLoaded=!configfs1733worker2 # [ 1.772796] systemd[1]: Queued start job for default target Initrd Default Target.1734lb # [ 1.760579] systemd[1]: Queued start job for default target Initrd Default Target.1735worker1 # [ 1.748722] systemd[1]: Mounting Kernel Configuration File System...1736worker2 # [ 1.783846] systemd[1]: Created slice Slice /system/modprobe.1737client # [ 1.744411] systemd[1]: Mounting Kernel Configuration File System...1738worker2 # [ 1.785028] systemd[1]: Started Dispatch Password Requests to Console Directory Watch.1739lb # [ 1.770092] systemd[1]: Created slice Slice /system/modprobe.1740worker2 # [ 1.786426] systemd[1]: Expecting device /dev/disk/by-label/nixos...1741worker2 # [ 1.787456] systemd[1]: Reached target Path Units.1742lb # [ 1.771294] systemd[1]: Started Dispatch Password Requests to Console Directory Watch.1743worker2 # [ 1.788259] systemd[1]: Reached target Slice Units.1744lb # [ 1.772547] systemd[1]: Expecting device /dev/disk/by-label/nixos...1745worker2 # [ 1.789062] systemd[1]: Reached target Swaps.1746lb # [ 1.773546] systemd[1]: Reached target Path Units.1747worker2 # [ 1.789841] systemd[1]: Reached target Timer Units.1748lb # [ 1.774322] systemd[1]: Reached target Slice Units.1749lb # [ 1.775095] systemd[1]: Reached target Swaps.1750worker2 # [ 1.790861] systemd[1]: Listening on D-Bus System Message Bus Socket.1751lb # [ 1.775885] systemd[1]: Reached target Timer Units.1752worker2 # [ 1.792094] systemd[1]: Listening on Journal Socket (/dev/log).1753lb # [ 1.776915] systemd[1]: Listening on D-Bus System Message Bus Socket.1754worker2 # [ 1.793267] systemd[1]: Listening on Journal Sockets.1755worker1 # [ 1.761534] systemd[1]: Starting Journal Service...1756lb # [ 1.778086] systemd[1]: Listening on Journal Socket (/dev/log).1757worker2 # [ 1.794233] systemd[1]: Listening on udev Control Socket.1758lb # [ 1.779245] systemd[1]: Listening on Journal Sockets.1759worker2 # [ 1.795326] systemd[1]: Listening on udev Kernel Socket.1760lb # [ 1.780213] systemd[1]: Listening on udev Control Socket.1761worker2 # [ 1.796217] systemd[1]: Reached target Socket Units.1762lb # [ 1.781179] systemd[1]: Listening on udev Kernel Socket.1763lb # [ 1.782012] systemd[1]: Reached target Socket Units.1764client # [ 1.760203] systemd[1]: Starting Journal Service...1765worker2 # [ 1.798910] systemd[1]: Starting Create List of Static Device Nodes...1766lb # [ 1.784453] systemd[1]: Starting Create List of Static Device Nodes...1767client # [ 1.768191] systemd[1]: Starting Load Kernel Modules...1768worker2 # [ 1.805801] systemd[1]: Starting Load Kernel Module 9pnet_virtio...1769client # [ 1.769015] systemd[1]: Early TPM SRK Setup skipped, unmet condition check ConditionSecurity=measured-os1770worker2 # [ 1.806943] systemd[1]: Load Kernel Module configfs skipped, unmet condition check ConditionKernelModuleLoaded=!configfs1771lb # [ 1.794830] systemd[1]: Starting Load Kernel Module 9pnet_virtio...1772worker1 # [ 1.780160] systemd[1]: Starting Load Kernel Modules...1773worker1 # [ 1.781144] systemd[1]: Early TPM SRK Setup skipped, unmet condition check ConditionSecurity=measured-os1774lb # [ 1.796314] systemd[1]: Load Kernel Module configfs skipped, unmet condition check ConditionKernelModuleLoaded=!configfs1775worker2 # [ 1.817383] systemd[1]: Mounting Kernel Configuration File System...1776client # [ 1.782178] systemd[1]: Starting Coldplug All udev Devices...1777lb # [ 1.804827] systemd[1]: Mounting Kernel Configuration File System...1778worker1 # [ 1.790971] systemd[1]: Starting Coldplug All udev Devices...1779worker2 # [ 1.827031] systemd[1]: Starting Journal Service...1780lb # [ 1.819436] systemd[1]: Starting Journal Service...1781worker1 # [ 1.813317] systemd[1]: Finished Create List of Static Device Nodes.1782worker1 # [ 1.814582] systemd[1]: modprobe@9pnet_virtio.service: Deactivated successfully.1783client # [ 1.808248] systemd[1]: Finished Create List of Static Device Nodes.1784worker1 # [ 1.814928] systemd[1]: Finished Load Kernel Module 9pnet_virtio.1785client # [ 1.809238] systemd[1]: modprobe@9pnet_virtio.service: Deactivated successfully.1786worker1 # [ 1.815271] systemd[1]: Mounted Kernel Configuration File System.1787client # [ 1.809551] systemd[1]: Finished Load Kernel Module 9pnet_virtio.1788client # [ 1.809860] systemd[1]: Mounted Kernel Configuration File System.1789worker1 # [ 1.825770] systemd[1]: Starting Create Static Device Nodes in /dev gracefully...1790lb # [ 1.840531] systemd[1]: Starting Load Kernel Modules...1791lb # [ 1.840663] systemd[1]: Early TPM SRK Setup skipped, unmet condition check ConditionSecurity=measured-os1792worker2 # [ 1.857745] systemd[1]: Starting Load Kernel Modules...1793worker2 # [ 1.857915] systemd[1]: Early TPM SRK Setup skipped, unmet condition check ConditionSecurity=measured-os1794client # [ 1.826059] device-mapper: core: CONFIG_IMA_DISABLE_HTABLE is disabled. Duplicate IMA measurements will not be recorded in the IMA log.1795client # [ 1.826768] systemd[1]: Starting Create Static Device Nodes in /dev gracefully...1796worker1 # [ 1.838031] device-mapper: core: CONFIG_IMA_DISABLE_HTABLE is disabled. Duplicate IMA measurements will not be recorded in the IMA log.1797lb # [ 1.855347] systemd[1]: Starting Coldplug All udev Devices...1798client # [ 1.838949] systemd-journald[73]: Collecting audit messages is disabled.1799worker2 # [ 1.877287] systemd[1]: Starting Coldplug All udev Devices...1800lb # [ 1.864497] systemd-journald[73]: Collecting audit messages is disabled.1801worker1 # [ 1.849635] device-mapper: ioctl: 4.50.0-ioctl (2025-04-28) initialised: dm-devel@lists.linux.dev1802client # [ 1.844141] device-mapper: ioctl: 4.50.0-ioctl (2025-04-28) initialised: dm-devel@lists.linux.dev1803worker1 # [ 1.851922] systemd-journald[73]: Collecting audit messages is disabled.1804lb # [ 1.868291] systemd[1]: Finished Create List of Static Device Nodes.1805lb # [ 1.870262] systemd[1]: modprobe@9pnet_virtio.service: Deactivated successfully.1806lb # [ 1.875291] systemd[1]: Finished Load Kernel Module 9pnet_virtio.1807lb # [ 1.875661] systemd[1]: Mounted Kernel Configuration File System.1808client # [ 1.853151] [drm] pci: virtio-gpu-pci detected at 0000:00:08.01809client # [ 1.853388] [drm] features: -virgl +edid -resource_blob -host_visible1810client # [ 1.853398] [drm] features: -context_init1811worker2 # [ 1.894498] device-mapper: core: CONFIG_IMA_DISABLE_HTABLE is disabled. Duplicate IMA measurements will not be recorded in the IMA log.1812client # [ 1.854129] [drm] number of scanouts: 11813client # [ 1.854147] [drm] number of cap sets: 01814worker2 # [ 1.896431] systemd[1]: Finished Create List of Static Device Nodes.1815worker1 # [ 1.865561] [drm] pci: virtio-gpu-pci detected at 0000:00:08.01816worker1 # [ 1.865800] [drm] features: -virgl +edid -resource_blob -host_visible1817worker1 # [ 1.865810] [drm] features: -context_init1818worker1 # [ 1.866529] [drm] number of scanouts: 11819worker2 # [ 1.901937] systemd[1]: modprobe@9pnet_virtio.service: Deactivated successfully.1820worker1 # [ 1.866548] [drm] number of cap sets: 01821worker2 # [ 1.903253] systemd[1]: Finished Load Kernel Module 9pnet_virtio.1822worker2 # [ 1.904689] systemd[1]: Mounted Kernel Configuration File System.1823worker2 # [ 1.907083] systemd[1]: Starting Create Static Device Nodes in /dev gracefully...1824worker2 # [ 1.910835] systemd-journald[73]: Collecting audit messages is disabled.1825worker2 # [ 1.911787] device-mapper: ioctl: 4.50.0-ioctl (2025-04-28) initialised: dm-devel@lists.linux.dev1826lb # [ 1.899720] systemd[1]: Starting Create Static Device Nodes in /dev gracefully...1827client # [ 1.876291] virtio-pci 0000:00:08.0: [drm] Registered 1 planes with drm panic1828client # [ 1.876318] [drm] Initialized virtio_gpu 0.1.0 for 0000:00:08.0 on minor 01829worker1 # [ 1.885668] virtio-pci 0000:00:08.0: [drm] Registered 1 planes with drm panic1830worker1 # [ 1.885696] [drm] Initialized virtio_gpu 0.1.0 for 0000:00:08.0 on minor 01831worker2 # [ 1.929540] [drm] pci: virtio-gpu-pci detected at 0000:00:08.01832worker2 # [ 1.929808] [drm] features: -virgl +edid -resource_blob -host_visible1833client # [ 1.892723] systemd[1]: Finished Create Static Device Nodes in /dev gracefully.1834worker2 # [ 1.929817] [drm] features: -context_init1835worker2 # [ 1.930615] [drm] number of scanouts: 11836worker2 # [ 1.930632] [drm] number of cap sets: 01837client # [ 1.896408] systemd[1]: Starting Create Static Device Nodes in /dev...1838worker1 # [ 1.901944] systemd[1]: Finished Create Static Device Nodes in /dev gracefully.1839worker1 # [ 1.904822] systemd[1]: Starting Create Static Device Nodes in /dev...1840lb # [ 1.919907] systemd[1]: Finished Create Static Device Nodes in /dev gracefully.1841lb # [ 1.931677] systemd[1]: Starting Create Static Device Nodes in /dev...1842worker1 # [ 1.913668] Console: switching to colour frame buffer device 160x501843worker2 # [ 1.949521] virtio-pci 0000:00:08.0: [drm] Registered 1 planes with drm panic1844worker1 # [ 1.920419] virtio-pci 0000:00:08.0: [drm] fb0: virtio_gpudrmfb frame buffer device1845client # [ 1.908773] Console: switching to colour frame buffer device 160x501846worker2 # [ 1.949550] [drm] Initialized virtio_gpu 0.1.0 for 0000:00:08.0 on minor 01847lb # [ 1.946829] device-mapper: core: CONFIG_IMA_DISABLE_HTABLE is disabled. Duplicate IMA measurements will not be recorded in the IMA log.1848worker2 # [ 1.970278] systemd[1]: Finished Create Static Device Nodes in /dev gracefully.1849client # [ 1.937053] virtio-pci 0000:00:08.0: [drm] fb0: virtio_gpudrmfb frame buffer device1850worker2 # [ 1.977447] systemd[1]: Starting Create Static Device Nodes in /dev...1851worker1 # [ 1.945462] systemd[1]: Finished Load Kernel Modules.1852worker1 # [ 1.949076] systemd[1]: Starting Apply Kernel Variables...1853lb # [ 1.963240] device-mapper: ioctl: 4.50.0-ioctl (2025-04-28) initialised: dm-devel@lists.linux.dev1854lb # [ 1.969432] [drm] pci: virtio-gpu-pci detected at 0000:00:08.01855lb # [ 1.969669] [drm] features: -virgl +edid -resource_blob -host_visible1856lb # [ 1.969679] [drm] features: -context_init1857lb # [ 1.970432] [drm] number of scanouts: 11858lb # [ 1.970449] [drm] number of cap sets: 01859client # [ 1.952712] systemd[1]: Finished Load Kernel Modules.1860worker2 # [ 1.985515] Console: switching to colour frame buffer device 160x501861client # [ 1.960503] systemd[1]: Starting Apply Kernel Variables...1862lb # [ 1.987912] systemd[1]: Finished Create Static Device Nodes in /dev.1863lb # [ 1.988168] systemd[1]: Reached target Preparation for Local File Systems.1864lb # [ 1.988197] systemd[1]: Reached target Local File Systems.1865lb # [ 1.992460] virtio-pci 0000:00:08.0: [drm] Registered 1 planes with drm panic1866client # [ 1.976284] systemd[1]: Finished Create Static Device Nodes in /dev.1867worker1 # [ 1.981591] systemd[1]: Finished Create Static Device Nodes in /dev.1868worker2 # [ 2.013761] virtio-pci 0000:00:08.0: [drm] fb0: virtio_gpudrmfb frame buffer device1869lb # [ 1.992484] [drm] Initialized virtio_gpu 0.1.0 for 0000:00:08.0 on minor 01870worker1 # [ 1.981838] systemd[1]: Reached target Preparation for Local File Systems.1871worker1 # [ 1.981874] systemd[1]: Reached target Local File Systems.1872client # [ 1.978517] systemd[1]: Reached target Preparation for Local File Systems.1873client # [ 1.978561] systemd[1]: Reached target Local File Systems.1874client # [ 1.982998] systemd[1]: Starting Rule-based Manager for Device Events and Files...1875worker1 # [ 1.989662] systemd[1]: Starting Rule-based Manager for Device Events and Files...1876lb # [ 2.001395] systemd[1]: Starting Rule-based Manager for Device Events and Files...1877worker2 # [ 2.030434] systemd[1]: Finished Load Kernel Modules.1878worker2 # [ 2.037695] systemd[1]: Starting Apply Kernel Variables...1879lb # [ 2.024489] Console: switching to colour frame buffer device 160x501880worker1 # [ 2.021665] systemd[1]: Finished Apply Kernel Variables.1881worker2 # [ 2.053614] systemd[1]: Finished Create Static Device Nodes in /dev.1882worker2 # [ 2.055959] systemd[1]: Reached target Preparation for Local File Systems.1883worker2 # [ 2.056018] systemd[1]: Reached target Local File Systems.1884client # [ 2.028437] systemd[1]: Finished Apply Kernel Variables.1885worker2 # [ 2.066747] systemd[1]: Starting Rule-based Manager for Device Events and Files...1886lb # [ 2.059819] virtio-pci 0000:00:08.0: [drm] fb0: virtio_gpudrmfb frame buffer device1887lb # [ 2.064900] systemd[1]: Finished Load Kernel Modules.1888client # [ 2.043420] systemd[1]: Started Journal Service.1889worker1 # [ 2.054900] systemd[1]: Started Journal Service.1890lb # [ 2.057331] systemd-modules-load[75]: Inserted module 'dm_mod'1891client # [ 2.036312] systemd-modules-load[74]: Inserted module 'dm_mod'1892lb # [ 2.075876] systemd[1]: Starting Apply Kernel Variables...1893client # [ 2.040347] systemd-modules-load[74]: Module 'virtio_balloon' is built in1894lb # [ 2.077860] systemd[1]: Started Journal Service.1895client # [ 2.041516] systemd-modules-load[74]: Module 'virtio_console' is built in1896client # [ 2.042622] systemd-modules-load[74]: Inserted module 'virtio_gpu'1897worker1 # [ 2.048377] systemd-modules-load[74]: Inserted module 'dm_mod'1898client # [ 2.043612] systemd-modules-load[74]: Module 'virtio_rng' is built in1899worker1 # [ 2.049609] systemd-modules-load[74]: Module 'virtio_balloon' is built in1900worker1 # [ 2.050729] systemd-modules-load[74]: Module 'virtio_console' is built in1901worker1 # [ 2.051818] systemd-modules-load[74]: Inserted module 'virtio_gpu'1902lb # [ 2.072265] systemd-modules-load[75]: Module 'virtio_balloon' is built in1903lb # [ 2.073511] systemd-modules-load[75]: Module 'virtio_console' is built in1904lb # [ 2.076188] systemd-modules-load[75]: Inserted module 'virtio_gpu'1905worker1 # [ 2.061467] systemd-modules-load[74]: Module 'virtio_rng' is built in1906worker1 # [ 2.062530] systemd[1]: Starting Create System Files and Directories...1907worker2 # [ 2.109553] systemd[1]: Finished Apply Kernel Variables.1908client # [ 2.057325] systemd[1]: Starting Create System Files and Directories...1909lb # [ 2.084334] systemd-modules-load[75]: Module 'virtio_rng' is built in1910worker2 # [ 2.118205] systemd[1]: Started Journal Service.1911worker1 # [ 2.077272] systemd-udevd[81]: Using default interface naming scheme 'v261'.1912worker2 # [ 2.113503] systemd-modules-load[74]: Inserted module 'dm_mod'1913worker2 # [ 2.114663] systemd-modules-load[74]: Module 'virtio_balloon' is built in1914worker2 # [ 2.115802] systemd-modules-load[74]: Module 'virtio_console' is built in1915client # [ 2.082284] systemd-udevd[81]: Using default interface naming scheme 'v261'.1916worker2 # [ 2.124444] systemd-modules-load[74]: Inserted module 'virtio_gpu'1917worker2 # [ 2.125462] systemd-modules-load[74]: Module 'virtio_rng' is built in1918lb # [ 2.108380] systemd-udevd[80]: Using default interface naming scheme 'v261'.1919worker2 # [ 2.126455] systemd[1]: Starting Create System Files and Directories...1920lb # [ 2.118093] systemd[1]: Starting Create System Files and Directories...1921client # [ 2.098218] systemd[1]: Finished Create System Files and Directories.1922worker1 # [ 2.106506] systemd[1]: Finished Create System Files and Directories.1923lb # [ 2.129960] systemd[1]: Finished Apply Kernel Variables.1924worker1 # [ 2.129205] systemd[1]: Started Rule-based Manager for Device Events and Files.1925client # [ 2.124388] systemd[1]: Started Rule-based Manager for Device Events and Files.1926lb # [ 2.149761] systemd[1]: Finished Create System Files and Directories.1927worker2 # [ 2.169184] systemd[1]: Finished Create System Files and Directories.1928worker2 # [ 2.173720] systemd-udevd[81]: Using default interface naming scheme 'v261'.1929lb # [ 2.162339] systemd[1]: Started Rule-based Manager for Device Events and Files.1930worker2 # [ 2.209433] systemd[1]: Started Rule-based Manager for Device Events and Files.1931worker1 # [ 2.194264] systemd[1]: Starting Virtual Console Setup...1932client # [ 2.196144] systemd[1]: Starting Virtual Console Setup...1933lb # [ 2.226407] systemd[1]: Starting Virtual Console Setup...1934worker2 # [ 2.276151] systemd[1]: Starting Virtual Console Setup...1935worker1 # [ 2.252489] systemd-vconsole-setup[103]: Configuration of first virtual console was skipped, ignoring remaining ones.1936client # [ 2.248554] systemd-vconsole-setup[103]: Configuration of first virtual console was skipped, ignoring remaining ones.1937client # [ 2.251316] systemd[1]: Finished Virtual Console Setup.1938worker1 # [ 2.260168] systemd[1]: Finished Virtual Console Setup.1939lb # [ 2.288548] systemd-vconsole-setup[103]: Configuration of first virtual console was skipped, ignoring remaining ones.1940lb # [ 2.292096] systemd[1]: Finished Virtual Console Setup.1941worker2 # [ 2.332510] systemd-vconsole-setup[103]: Configuration of first virtual console was skipped, ignoring remaining ones.1942worker2 # [ 2.335359] systemd[1]: Finished Virtual Console Setup.1943lb # [ 2.950291] systemd[1]: Finished Coldplug All udev Devices.1944lb # [ 2.951953] systemd[1]: Reached target System Initialization.1945lb # [ 2.956148] systemd[1]: Reached target Basic System.1946worker1 # [ 2.940476] systemd[1]: Finished Coldplug All udev Devices.1947worker1 # [ 2.941829] systemd[1]: Reached target System Initialization.1948worker1 # [ 2.943177] systemd[1]: Reached target Basic System.1949client # [ 2.968488] systemd[1]: Finished Coldplug All udev Devices.1950client # [ 2.970215] systemd[1]: Reached target System Initialization.1951client # [ 2.971038] systemd[1]: Reached target Basic System.1952worker2 # [ 3.022531] systemd[1]: Finished Coldplug All udev Devices.1953worker2 # [ 3.023548] systemd[1]: Reached target System Initialization.1954worker2 # [ 3.024451] systemd[1]: Reached target Basic System.1955worker1 # [ 3.085996] (udev-worker)[106]: Network interface NamePolicy= disabled on kernel command line.1956client # [ 3.104622] (udev-worker)[98]: eth1: Config file /etc/systemd/network/40-eth1.link is applied to device based on potentially unpredictable interface name.1957lb # [ 3.129936] (udev-worker)[95]: Network interface NamePolicy= disabled on kernel command line.1958worker1 # [ 3.122345] (udev-worker)[94]: eth1: Config file /etc/systemd/network/40-eth1.link is applied to device based on potentially unpredictable interface name.1959lb # [ 3.139360] (udev-worker)[97]: eth1: Config file /etc/systemd/network/40-eth1.link is applied to device based on potentially unpredictable interface name.1960lb # [ 3.144312] (udev-worker)[97]: Network interface NamePolicy= disabled on kernel command line.1961worker1 # [ 3.128282] (udev-worker)[94]: Network interface NamePolicy= disabled on kernel command line.1962worker2 # [ 3.171864] (udev-worker)[95]: eth1: Config file /etc/systemd/network/40-eth1.link is applied to device based on potentially unpredictable interface name.1963client # [ 3.152082] (udev-worker)[98]: Network interface NamePolicy= disabled on kernel command line.1964client # [ 3.155341] (udev-worker)[106]: Network interface NamePolicy= disabled on kernel command line.1965worker2 # [ 3.213098] (udev-worker)[96]: Network interface NamePolicy= disabled on kernel command line.1966worker2 # [ 3.224160] (udev-worker)[95]: Network interface NamePolicy= disabled on kernel command line.1967lb # [ 3.208938] systemd[1]: Found device /dev/disk/by-label/nixos.1968lb # [ 3.211588] systemd[1]: Reached target Initrd Root Device.1969lb # [ 3.216143] systemd[1]: Starting File System Check on /dev/disk/by-label/nixos...1970worker1 # [ 3.209026] systemd[1]: Found device /dev/disk/by-label/nixos.1971worker1 # [ 3.213382] systemd[1]: Reached target Initrd Root Device.1972worker1 # [ 3.214735] systemd[1]: Starting File System Check on /dev/disk/by-label/nixos...1973client # [ 3.219867] systemd[1]: Found device /dev/disk/by-label/nixos.1974client # [ 3.227538] systemd[1]: Reached target Initrd Root Device.1975client # [ 3.233259] systemd[1]: Starting File System Check on /dev/disk/by-label/nixos...1976lb # [ 3.271267] systemd-fsck[112]: nixos: clean, 12/65536 files, 13019/262144 blocks1977worker2 # [ 3.289497] systemd[1]: Found device /dev/disk/by-label/nixos.1978worker2 # [ 3.291616] systemd[1]: Reached target Initrd Root Device.1979worker2 # [ 3.295654] systemd[1]: Starting File System Check on /dev/disk/by-label/nixos...1980lb # [ 3.278038] systemd[1]: Finished File System Check on /dev/disk/by-label/nixos.1981worker1 # [ 3.264810] systemd-fsck[112]: nixos: clean, 12/65536 files, 13019/262144 blocks1982lb # [ 3.282955] systemd[1]: Mounting /sysroot...1983worker1 # [ 3.274254] systemd[1]: Finished File System Check on /dev/disk/by-label/nixos.1984worker1 # [ 3.276641] systemd[1]: Mounting /sysroot...1985client # [ 3.293567] systemd-fsck[112]: nixos: clean, 12/65536 files, 13019/262144 blocks1986lb # [ 3.333318] EXT4-fs (vda): mounted filesystem 41855766-2557-42a8-8a8c-02099e3fa247 r/w with ordered data mode. Quota mode: none.1987client # [ 3.300104] systemd[1]: Finished File System Check on /dev/disk/by-label/nixos.1988lb # [ 3.324806] systemd[1]: Mounted /sysroot.1989client # [ 3.303019] systemd[1]: Mounting /sysroot...1990lb # [ 3.325894] systemd[1]: Reached target Initrd Root File System.1991lb # [ 3.329595] systemd[1]: Starting Mountpoints Configured in the Real Root...1992worker1 # [ 3.330897] EXT4-fs (vda): mounted filesystem 09dc7d4e-8077-4e0f-bde3-c958c06e6d21 r/w with ordered data mode. Quota mode: none.1993worker1 # [ 3.319104] systemd[1]: Mounted /sysroot.1994worker1 # [ 3.321111] systemd[1]: Reached target Initrd Root File System.1995worker1 # [ 3.322591] systemd[1]: Starting Mountpoints Configured in the Real Root...1996worker2 # [ 3.360626] systemd-fsck[112]: nixos: clean, 12/65536 files, 13019/262144 blocks1997worker2 # [ 3.366471] systemd[1]: Finished File System Check on /dev/disk/by-label/nixos.1998worker2 # [ 3.373809] systemd[1]: Mounting /sysroot...1999lb # [ 3.362601] systemd-sysroot-fstab-check[120]: /sysroot should be mounted in the initrd, will request daemon-reload.2000worker1 # [ 3.348109] systemd-sysroot-fstab-check[120]: /sysroot should be mounted in the initrd, will request daemon-reload.
loading 2,018 hidden lines…
4019worker2 # [ 38.432277] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4020worker1 # [ 38.460264] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514021worker1 # [ 38.465978] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4022lb: must succeed: curl -sf localhost:9901/clusters4023worker2 # [ 38.634187] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514024lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.11 seconds)4025worker2 # [ 38.640820] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4026worker1 # [ 38.669807] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514027worker1 # [ 38.674023] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4028worker2 # [ 38.845826] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514029worker2 # [ 38.850296] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4030worker1 # [ 38.870793] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514031worker1 # [ 38.876797] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4032worker2 # [ 39.046987] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514033worker2 # [ 39.052997] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4034worker1 # [ 39.078752] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514035worker1 # [ 39.090658] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4036worker2 # [ 39.253857] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514037worker2 # [ 39.256991] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4038worker1 # [ 39.288834] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_connected addr=lb:500514039worker1 # [ 39.291753] nix-grpc-daemon[777]: ts=2026-09-22T10:36:35Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4040worker2 # [ 39.457877] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514041worker2 # [ 39.465281] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4042worker1 # [ 39.493122] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514043worker1 # [ 39.498868] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4044lb: must succeed: curl -sf localhost:9901/clusters4045worker2 # [ 39.666979] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514046worker2 # [ 39.673416] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4047lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)4048worker1 # [ 39.702199] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514049worker1 # [ 39.706973] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4050worker2 # [ 39.876532] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514051worker2 # [ 39.880724] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4052worker1 # [ 39.902734] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514053worker1 # [ 39.908937] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4054worker2 # [ 40.079083] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514055worker2 # [ 40.085575] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4056worker1 # [ 40.109958] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514057worker1 # [ 40.116593] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4058worker2 # [ 40.286594] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514059worker2 # [ 40.291891] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4060worker1 # [ 40.318249] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_connected addr=lb:500514061worker1 # [ 40.323745] nix-grpc-daemon[777]: ts=2026-09-22T10:36:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4062worker2 # [ 40.495527] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514063worker2 # [ 40.501756] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4064worker1 # [ 40.525442] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514065worker1 # [ 40.531259] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4066worker2 # [ 40.703526] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514067worker2 # [ 40.709811] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4068lb: must succeed: curl -sf localhost:9901/clusters4069worker1 # [ 40.734215] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514070worker1 # [ 40.740953] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4071lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.12 seconds)4072worker2 # [ 40.912874] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514073worker2 # [ 40.918338] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4074worker1 # [ 40.941926] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514075worker1 # [ 40.947188] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4076worker2 # [ 41.119520] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514077worker2 # [ 41.124711] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4078worker1 # [ 41.147893] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514079worker1 # [ 41.153041] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4080worker2 # [ 41.325537] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514081worker2 # [ 41.330539] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4082worker1 # [ 41.353974] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_connected addr=lb:500514083worker1 # [ 41.358994] nix-grpc-daemon[777]: ts=2026-09-22T10:36:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4084worker2 # [ 41.531685] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514085worker2 # [ 41.536945] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4086worker1 # [ 41.559876] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514087worker1 # [ 41.565028] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4088worker2 # [ 41.737846] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514089worker2 # [ 41.742820] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4090worker1 # [ 41.767487] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514091worker1 # [ 41.771761] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4092lb: must succeed: curl -sf localhost:9901/clusters4093lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)4094worker2 # [ 41.945546] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514095worker2 # [ 41.949595] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4096worker1 # [ 41.967787] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514097worker1 # [ 41.972983] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4098worker2 # [ 42.146430] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514099worker2 # [ 42.151466] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4100worker2 # [ 42.202747] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_take_over4101lb # [ 42.187226] niks3-server[1035]: 2026/09/22 10:36:38 INFO lead: acquired remote=192.168.1.4:407504102worker1 # [ 42.173791] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514103worker1 # [ 42.178778] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"4104worker2 # [ 42.352456] nix-grpc-daemon[777]: ts=2026-09-22T10:36:38Z level=info event=scheduler_connected addr=lb:500514105worker2 # [ 42.379184] nix-grpc-daemon[777]: ts=2026-09-22T10:36:39Z level=info event=oidc_jwks provider=mock keys=14106worker2 # [ 42.384166] nix-grpc-daemon[777]: ts=2026-09-22T10:36:39Z level=info event=worker_session_open cn=oidc:mock:node:worker2 peer=ipv4:192.168.1.2:411284107worker2 # [ 42.389404] nix-grpc-daemon[777]: ts=2026-09-22T10:36:39Z level=info event=worker_hello addr=192.168.1.4:50051 systems=aarch64-linux max_jobs=1 running=04108worker1 # [ 42.384781] nix-grpc-daemon[777]: ts=2026-09-22T10:36:39Z level=info event=scheduler_connected addr=lb:500514109worker2 # [ 42.421821] nix-grpc-daemon[777]: ts=2026-09-22T10:36:39Z level=info event=worker_session_open cn=worker-1 peer=ipv4:192.168.1.2:411284110worker2 # [ 42.426799] nix-grpc-daemon[777]: ts=2026-09-22T10:36:39Z level=info event=worker_hello addr=192.168.1.3:50051 systems=aarch64-linux max_jobs=1 running=04111lb: must succeed: curl -sf localhost:9901/clusters4112lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)4113worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics4114worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)4115lb: must succeed: curl -sf localhost:9901/clusters4116lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.07 seconds)4117lb: must succeed: curl -sf localhost:9901/clusters4118lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)4119lb: must succeed: curl -sf localhost:9901/clusters4120lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)4121lb: must succeed: curl -sf localhost:9901/clusters4122lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.04 seconds)4123worker2: must succeed: journalctl -u nix-grpc-daemon -o cat | grep 'event=worker_session_open'4124worker2: (finished: must succeed: journalctl -u nix-grpc-daemon -o cat | grep 'event=worker_session_open', in 0.07 seconds)4125worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics4126worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.06 seconds)4127lb: must succeed: curl -sf localhost:9901/clusters4128lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)4129lb: must succeed: curl -sf localhost:9901/clusters4130lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)4131worker1: must succeed: curl -sf http://127.0.0.1:9464/metrics4132worker1: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.09 seconds)4133worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics4134worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)4135worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics4136worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.06 seconds)4137worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics4138worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.04 seconds)4139(finished: subtest: one node schedules and both builders hold a WorkerSession on it, in 10.70 seconds)4140subtest: balancer auth: client cert, bearer token, nothing, unknown CN4141worker2 # [ 44.245219] nix-grpc-daemon[777]: ts=2026-09-22T10:36:40Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:411444142worker2 # [ 44.264871] systemd[1]: Started Nix Daemon.4143worker2 # [ 44.536190] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4144worker2 # [ 44.561804] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 93 ms (attempt 1/5)4145worker2 # [ 44.657568] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 169 ms (attempt 2/5)4146worker2 # [ 44.826840] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 280 ms (attempt 3/5)4147worker2 # [ 45.114756] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 425 ms (attempt 4/5)4148worker2 # [ 45.550497] nix-grpc-daemon[777]: disabling binary cache 'https://cache.nixos.org' for 60 seconds4149worker2 # [ 45.556799] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org4150worker1 # [ 45.556828] nix-grpc-daemon[777]: ts=2026-09-22T10:36:42Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:330684151worker1 # [ 45.577415] systemd[1]: Started Nix Daemon.4152worker1 # [ 45.918502] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4153worker1 # [ 45.950228] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 82 ms (attempt 1/5)4154worker1 # [ 46.035093] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 12 ms (attempt 2/5)4155worker1 # [ 46.054588] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 276 ms (attempt 3/5)4156worker1 # [ 46.338160] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 45 ms (attempt 4/5)4157worker1 # [ 46.385274] nix-grpc-daemon[777]: disabling binary cache 'https://cache.nixos.org' for 60 seconds4158worker1 # [ 46.390601] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org4159worker1 # [ 46.403450] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=path_info_miss path=r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test error="error: path '/nix/store/r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test' is required, but there is no substituter that can build it"4160worker1 # [ 46.417078] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:330684161worker1 # [ 46.427635] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=path_info_miss path=r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test error="error: path '/nix/store/r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test' is required, but there is no substituter that can build it"4162client: must succeed: curl -sfG http://lb:8081/issue --data-urlencode 'aud=grpc://lb:50051' --data-urlencode sub=dev:alice > /root/dev.jwt && test -s /root/dev.jwt4163client: (finished: must succeed: curl -sfG http://lb:8081/issue --data-urlencode 'aud=grpc://lb:50051' --data-urlencode sub=dev:alice > /root/dev.jwt && test -s /root/dev.jwt, in 0.10 seconds)4164client: must succeed: install -D /nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/foreign.pem /var/lib/nix-grpc-store/client.crt && install -D /nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/foreign.key /var/lib/nix-grpc-store/client.key4165client: (finished: must succeed: install -D /nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/foreign.pem /var/lib/nix-grpc-store/client.crt && install -D /nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/foreign.key /var/lib/nix-grpc-store/client.key, in 0.04 seconds)4166worker2 # [ 46.891788] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=rpc_start method=QueryMissing cn=oidc:mock:dev:alice peer=ipv4:192.168.1.2:411444167worker2 # [ 46.911115] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=rpc_start method=QueryPathInfos cn=oidc:mock:dev:alice peer=ipv4:192.168.1.2:411444168worker2 # [ 46.926207] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=path_info_miss path=r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test error="error: path '/nix/store/r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test' is required, but there is no substituter that can build it"4169worker2 # [ 46.939307] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=rpc_start method=QueryPathInfos cn=oidc:mock:dev:alice peer=ipv4:192.168.1.2:411444170worker2 # [ 46.950747] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=debug event=path_info_miss path=r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test error="error: path '/nix/store/r2598nmbi6ac9dkryhsinmmd413x6pzp-nixos-system-client-test' is required, but there is no substituter that can build it"4171client: must succeed: rm -r /var/lib/nix-grpc-store4172client: (finished: must succeed: rm -r /var/lib/nix-grpc-store, in 0.04 seconds)4173worker1 # [ 47.227376] nix-grpc-daemon[777]: ts=2026-09-22T10:36:43Z level=info event=denied method=QueryMissing cn=- role=none4174worker1 # [ 47.482991] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=info event=denied method=QueryMissing cn=stranger role=none4175(finished: subtest: balancer auth: client cert, bearer token, nothing, unknown CN, in 3.69 seconds)4176subtest: DAG build is scheduled across workers and lands in the cache4177client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 >&24178worker1 # [ 47.731878] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:330684179client # don't know how to build these paths:4180client # /nix/store/czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv4181worker2 # [ 47.777657] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444182worker2 # [ 47.815193] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=debug event=want client=1 drv=619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv cached=04183worker2 # [ 47.822423] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=debug event=assigned drv=619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv worker=192.168.1.4:50051 clients=14184worker2 # [ 47.873215] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444185worker2 # [ 47.897710] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444186client # copying 1 paths to 'lb:50051'...4187worker2 # [ 47.913445] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:411444188client # copying path '/nix/store/619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv' to 'lb:50051'...4189worker2 # [ 47.939207] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4190worker2 # [ 47.977858] nix-grpc-daemon[777]: ts=2026-09-22T10:36:44Z level=info event=niks3_push_started pid=8794191worker2 # [ 48.255419] nix-grpc-daemon[879]: time=2026-09-22T10:36:44.898Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.key ca=""4192worker2 # [ 48.446384] nix-daemon[864]: accepted connection from pid 892, user nix-grpc-daemon (trusted)4193worker2 # [ 48.463428] nix-daemon[864]: reaped child process 895, status = succeeded4194worker2 # [ 48.638023] nix-daemon[864]: accepted connection from pid 897, user nix-grpc-daemon (trusted)4195worker2 # [ 48.655604] nix-daemon[864]: reaped child process 900, status = succeeded4196lb # [ 48.653275] niks3-server[1035]: 2026/09/22 10:36:45 INFO Received uploads request method=POST path=/api/pending_closures4197worker2 # [ 48.698090] nix-grpc-daemon[879]: time=2026-09-22T10:36:45.342Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4198worker2 # [ 48.701374] nix-grpc-daemon[879]: time=2026-09-22T10:36:45.344Z level=INFO msg="Uploading 619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv (504B)"4199lb # [ 48.733352] niks3-server[1035]: 2026/09/22 10:36:45 INFO Received sign narinfos request method=POST path=/api/pending_closures/1/sign4200lb # [ 48.745420] niks3-server[1035]: 2026/09/22 10:36:45 INFO Registered completed upload object_key=nar/1il50j4vra11xfnqgjhjgrjn1753frlhxnss6mh1dmp5ww3is34w.nar.zst4201lb # [ 48.785748] niks3-server[1035]: 2026/09/22 10:36:45 INFO Signed narinfos id=1 count=14202worker2 # [ 48.807341] nix-grpc-daemon[879]: time=2026-09-22T10:36:45.450Z level=INFO msg="Uploading 1 narinfos"4203lb # [ 48.803282] niks3-server[1035]: 2026/09/22 10:36:45 INFO Registered completed upload object_key=619ccx60pfv52kv4hhgqpj1ivdmfcr5r.ls4204lb # [ 48.810387] niks3-server[1035]: 2026/09/22 10:36:45 INFO Received complete upload request method=POST path=/api/pending_closures/1/complete4205lb # [ 48.821437] niks3-server[1035]: 2026/09/22 10:36:45 INFO Registered completed upload object_key=619ccx60pfv52kv4hhgqpj1ivdmfcr5r.narinfo4206lb # [ 48.829442] niks3-server[1035]: 2026/09/22 10:36:45 INFO Completed upload id=14207worker2 # [ 48.847897] nix-grpc-daemon[879]: time=2026-09-22T10:36:45.492Z level=INFO msg="Upload complete. (584ms)"4208worker2 # [ 48.860351] nix-grpc-daemon[777]: ts=2026-09-22T10:36:45Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_in=5044209worker2 # [ 48.873095] nix-grpc-daemon[777]: ts=2026-09-22T10:36:45Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444210worker2 # [ 48.887490] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4211worker2 # [ 48.900447] nix-daemon[864]: reaped child process 877, status = succeeded4212client # building '/nix/store/619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv' on lb:50051...4213client # farm-a> worker2: building 619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv4214worker2 # [ 48.912789] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4215client # farm-a> disabling binary cache 'https://cache.nixos.org' for 60 seconds4216client # farm-a> LOG-farm-a-t14217lb # [ 49.806555] envoy[896]: conn peer=[2001:db8:1::1]:32804 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=732 tx=407 ms=22624218lb # [ 49.812341] envoy[896]: conn peer=[2001:db8:1::1]:32816 tls=TLSv1.3 subject="-" sni=- flags=- tls_fail="-" rx=2596 tx=361 ms=784219lb # [ 49.817078] envoy[896]: conn peer=[2001:db8:1::1]:32820 tls=TLSv1.3 subject="-" sni=- flags=- tls_fail="-" rx=423 tx=198 ms=114220lb # [ 49.822063] envoy[896]: conn peer=[2001:db8:1::1]:32822 tls=TLSv1.3 subject="CN=stranger" sni=- flags=- tls_fail="-" rx=423 tx=181 ms=124221worker2 # [ 49.957880] nix-daemon[864]: reaped child process 904, status = succeeded4222worker2 # [ 50.131618] nix-daemon[864]: accepted connection from pid 920, user nix-grpc-daemon (trusted)4223worker2 # [ 50.147179] nix-daemon[864]: reaped child process 923, status = succeeded4224lb # [ 50.145325] niks3-server[1035]: 2026/09/22 10:36:46 INFO Received uploads request method=POST path=/api/pending_closures4225worker2 # [ 50.176422] nix-grpc-daemon[879]: time=2026-09-22T10:36:46.818Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4226worker2 # [ 50.178865] nix-grpc-daemon[879]: time=2026-09-22T10:36:46.819Z level=INFO msg="Uploading dg404zlvxfrspnxl6cc1qkc6i83m5a3a-farm-a (128B)"4227lb # [ 50.197178] niks3-server[1035]: 2026/09/22 10:36:46 INFO Registered completed upload object_key=nar/1rmil4ja9jzfwn6l5hwbir70b6plx99sgk0pnjjy377ngbc3vz28.nar.zst4228worker2 # [ 50.230137] nix-grpc-daemon[879]: time=2026-09-22T10:36:46.873Z level=INFO msg="Uploading 1 narinfos"4229lb # [ 50.213935] niks3-server[1035]: 2026/09/22 10:36:46 INFO Registered completed upload object_key=log/619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv4230lb # [ 50.222561] niks3-server[1035]: 2026/09/22 10:36:46 INFO Received sign narinfos request method=POST path=/api/pending_closures/2/sign4231lb # [ 50.233720] niks3-server[1035]: 2026/09/22 10:36:46 INFO Signed narinfos id=2 count=14232worker2 # [ 50.251863] nix-grpc-daemon[879]: time=2026-09-22T10:36:46.896Z level=INFO msg="Upload complete. (324ms)"4233lb # [ 50.236364] niks3-server[1035]: 2026/09/22 10:36:46 INFO Registered completed upload object_key=dg404zlvxfrspnxl6cc1qkc6i83m5a3a.ls4234lb # [ 50.239306] niks3-server[1035]: 2026/09/22 10:36:46 INFO Received complete upload request method=POST path=/api/pending_closures/2/complete4235lb # [ 50.243455] niks3-server[1035]: 2026/09/22 10:36:46 INFO Completed upload id=24236lb # [ 50.245980] niks3-server[1035]: 2026/09/22 10:36:46 INFO Registered completed upload object_key=dg404zlvxfrspnxl6cc1qkc6i83m5a3a.narinfo4237worker2 # [ 50.276392] nix-grpc-daemon[777]: ts=2026-09-22T10:36:46Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=1 drv=619ccx60pfv52kv4hhgqpj1ivdmfcr5r-farm-a.drv assign_id=1 outputs=14238worker2 # [ 50.290012] nix-grpc-daemon[777]: ts=2026-09-22T10:36:46Z level=debug event=want client=1 drv=79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv cached=04239worker2 # [ 50.293493] nix-grpc-daemon[777]: ts=2026-09-22T10:36:46Z level=debug event=assigned drv=79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv worker=192.168.1.4:50051 clients=14240worker2 # [ 50.296937] nix-daemon[864]: reaped child process 902, status = succeeded4241worker2 # [ 50.322772] nix-grpc-daemon[777]: ts=2026-09-22T10:36:46Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444242worker2 # [ 50.335467] nix-grpc-daemon[777]: ts=2026-09-22T10:36:46Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444243worker2 # [ 50.351137] nix-grpc-daemon[879]: time=2026-09-22T10:36:46.995Z level=INFO msg="All 1 paths already cached"4244client # copying 1 paths to 'lb:50051'...4245worker2 # [ 50.361589] nix-grpc-daemon[777]: ts=2026-09-22T10:36:47Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:411444246client # copying path '/nix/store/79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv' to 'lb:50051'...4247worker2 # [ 50.384529] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4248worker2 # [ 50.596805] nix-daemon[864]: accepted connection from pid 929, user nix-grpc-daemon (trusted)4249worker2 # [ 50.616255] nix-daemon[864]: reaped child process 932, status = succeeded4250worker2 # [ 50.789860] nix-daemon[864]: accepted connection from pid 934, user nix-grpc-daemon (trusted)4251worker2 # [ 50.808254] nix-daemon[864]: reaped child process 937, status = succeeded4252lb # [ 50.801805] niks3-server[1035]: 2026/09/22 10:36:47 INFO Received uploads request method=POST path=/api/pending_closures4253worker2 # [ 50.832935] nix-grpc-daemon[879]: time=2026-09-22T10:36:47.476Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"4254worker2 # [ 50.834372] nix-grpc-daemon[879]: time=2026-09-22T10:36:47.476Z level=INFO msg="Uploading 79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv (624B)"4255lb # [ 50.835998] niks3-server[1035]: 2026/09/22 10:36:47 INFO Registered completed upload object_key=nar/1xk4kbz8l387hfrqq2xi6as79has336h62f2ra95472yqjclksjg.nar.zst4256worker2 # [ 50.864209] nix-grpc-daemon[879]: time=2026-09-22T10:36:47.508Z level=INFO msg="Uploading 1 narinfos"4257lb # [ 50.851399] niks3-server[1035]: 2026/09/22 10:36:47 INFO Received sign narinfos request method=POST path=/api/pending_closures/3/sign4258lb # [ 50.861975] niks3-server[1035]: 2026/09/22 10:36:47 INFO Signed narinfos id=3 count=14259lb # [ 50.872607] niks3-server[1035]: 2026/09/22 10:36:47 INFO Registered completed upload object_key=79z38hfi82zcmb9yq1gcmfviimgf6z2l.ls4260lb # [ 50.875853] niks3-server[1035]: 2026/09/22 10:36:47 INFO Received complete upload request method=POST path=/api/pending_closures/3/complete4261lb # [ 50.880167] niks3-server[1035]: 2026/09/22 10:36:47 INFO Completed upload id=34262worker2 # [ 50.898413] nix-grpc-daemon[879]: time=2026-09-22T10:36:47.542Z level=INFO msg="Upload complete. (496ms)"4263lb # [ 50.884592] niks3-server[1035]: 2026/09/22 10:36:47 INFO Registered completed upload object_key=79z38hfi82zcmb9yq1gcmfviimgf6z2l.narinfo4264worker2 # [ 50.904230] nix-grpc-daemon[777]: ts=2026-09-22T10:36:47Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_in=6244265worker2 # [ 50.918156] nix-grpc-daemon[777]: ts=2026-09-22T10:36:47Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444266worker2 # [ 50.924375] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4267worker2 # [ 50.929096] nix-daemon[864]: reaped child process 927, status = succeeded4268client # building '/nix/store/79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv' on lb:50051...4269worker2 # [ 50.940337] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4270client # farm-b> worker2: building 79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv4271client # farm-b> disabling binary cache 'https://cache.nixos.org' for 60 seconds4272client # farm-b> LOG-farm-b-t14273worker2 # [ 52.192176] nix-daemon[864]: reaped child process 941, status = succeeded4274worker2 # [ 52.368993] nix-daemon[864]: accepted connection from pid 957, user nix-grpc-daemon (trusted)4275worker2 # [ 52.386052] nix-daemon[864]: reaped child process 960, status = succeeded4276lb # [ 52.381486] niks3-server[1035]: 2026/09/22 10:36:49 INFO Received uploads request method=POST path=/api/pending_closures4277worker2 # [ 52.410165] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.053Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"4278worker2 # [ 52.411697] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.053Z level=INFO msg="Uploading 8n9g36vw9957r150a7wg1k6yj9fbqm8r-farm-b (176B)"4279lb # [ 52.414843] niks3-server[1035]: 2026/09/22 10:36:49 INFO Registered completed upload object_key=log/79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv4280lb # [ 52.428871] niks3-server[1035]: 2026/09/22 10:36:49 INFO Registered completed upload object_key=nar/1mx8h9irf9ykfzxs6q3lwxbzxwv5d2s9mh2x0sky7448mlc1qxrx.nar.zst4281lb # [ 52.441475] niks3-server[1035]: 2026/09/22 10:36:49 INFO Received sign narinfos request method=POST path=/api/pending_closures/4/sign4282worker2 # [ 52.465492] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.109Z level=INFO msg="Uploading 1 narinfos"4283lb # [ 52.450525] niks3-server[1035]: 2026/09/22 10:36:49 INFO Signed narinfos id=4 count=14284lb # [ 52.455970] niks3-server[1035]: 2026/09/22 10:36:49 INFO Registered completed upload object_key=8n9g36vw9957r150a7wg1k6yj9fbqm8r.ls4285lb # [ 52.464325] niks3-server[1035]: 2026/09/22 10:36:49 INFO Received complete upload request method=POST path=/api/pending_closures/4/complete4286lb # [ 52.472570] niks3-server[1035]: 2026/09/22 10:36:49 INFO Completed upload id=44287worker2 # [ 52.490687] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.134Z level=INFO msg="Upload complete. (337ms)"4288lb # [ 52.475334] niks3-server[1035]: 2026/09/22 10:36:49 INFO Registered completed upload object_key=8n9g36vw9957r150a7wg1k6yj9fbqm8r.narinfo4289worker2 # [ 52.512813] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=1 drv=79z38hfi82zcmb9yq1gcmfviimgf6z2l-farm-b.drv assign_id=2 outputs=14290worker2 # [ 52.524340] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=debug event=want client=1 drv=czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv cached=04291worker2 # [ 52.530284] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=debug event=assigned drv=czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv worker=192.168.1.4:50051 clients=14292worker2 # [ 52.534218] nix-daemon[864]: reaped child process 939, status = succeeded4293worker2 # [ 52.554791] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444294worker2 # [ 52.569310] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444295worker2 # [ 52.584475] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.228Z level=INFO msg="All 2 paths already cached"4296client # copying 1 paths to 'lb:50051'...4297client # copying path '/nix/store/czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv' to 'lb:50051'...4298worker2 # [ 52.594363] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:411444299worker2 # [ 52.613556] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4300worker2 # [ 52.809458] nix-daemon[864]: accepted connection from pid 966, user nix-grpc-daemon (trusted)4301worker2 # [ 52.827480] nix-daemon[864]: reaped child process 969, status = succeeded4302worker2 # [ 52.998564] nix-daemon[864]: accepted connection from pid 971, user nix-grpc-daemon (trusted)4303worker2 # [ 53.017423] nix-daemon[864]: reaped child process 974, status = succeeded4304lb # [ 53.011618] niks3-server[1035]: 2026/09/22 10:36:49 INFO Received uploads request method=POST path=/api/pending_closures4305worker2 # [ 53.040448] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.684Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"4306worker2 # [ 53.041951] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.684Z level=INFO msg="Uploading czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv (752B)"4307lb # [ 53.041312] niks3-server[1035]: 2026/09/22 10:36:49 INFO Registered completed upload object_key=nar/02rh0zwkcxgd7rc74r5h676rr0m8dbayk3glb776b2mdlirxh9lv.nar.zst4308lb # [ 53.053246] niks3-server[1035]: 2026/09/22 10:36:49 INFO Received sign narinfos request method=POST path=/api/pending_closures/5/sign4309worker2 # [ 53.078986] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.723Z level=INFO msg="Uploading 1 narinfos"4310lb # [ 53.060833] niks3-server[1035]: 2026/09/22 10:36:49 INFO Signed narinfos id=5 count=14311lb # [ 53.067018] niks3-server[1035]: 2026/09/22 10:36:49 INFO Registered completed upload object_key=czd6ljynjaw3bl3w8iz1nx68whkdis8f.ls4312lb # [ 53.078030] niks3-server[1035]: 2026/09/22 10:36:49 INFO Received complete upload request method=POST path=/api/pending_closures/5/complete4313lb # [ 53.089262] niks3-server[1035]: 2026/09/22 10:36:49 INFO Completed upload id=54314worker2 # [ 53.107556] nix-grpc-daemon[879]: time=2026-09-22T10:36:49.751Z level=INFO msg="Upload complete. (477ms)"4315lb # [ 53.092919] niks3-server[1035]: 2026/09/22 10:36:49 INFO Registered completed upload object_key=czd6ljynjaw3bl3w8iz1nx68whkdis8f.narinfo4316worker2 # [ 53.116241] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_in=7524317worker2 # [ 53.124736] nix-grpc-daemon[777]: ts=2026-09-22T10:36:49Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444318worker2 # [ 53.134128] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4319worker2 # [ 53.138664] nix-daemon[864]: reaped child process 964, status = succeeded4320client # building '/nix/store/czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv' on lb:50051...4321client # farm-top> worker2: building czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv4322worker2 # [ 53.151229] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4323client # farm-top> disabling binary cache 'https://cache.nixos.org' for 60 seconds4324client # farm-top> LOG-farm-top-t14325worker2 # [ 54.173176] nix-daemon[864]: reaped child process 978, status = succeeded4326worker2 # [ 54.336970] nix-daemon[864]: accepted connection from pid 994, user nix-grpc-daemon (trusted)4327worker2 # [ 54.355834] nix-daemon[864]: reaped child process 997, status = succeeded4328lb # [ 54.350554] niks3-server[1035]: 2026/09/22 10:36:50 INFO Received uploads request method=POST path=/api/pending_closures4329worker2 # [ 54.380109] nix-grpc-daemon[879]: time=2026-09-22T10:36:51.022Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"4330worker2 # [ 54.381569] nix-grpc-daemon[879]: time=2026-09-22T10:36:51.022Z level=INFO msg="Uploading nblg066ygaz2avzlmix3rnngr62hld8f-farm-top (232B)"4331lb # [ 54.383715] niks3-server[1035]: 2026/09/22 10:36:51 INFO Registered completed upload object_key=log/czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv4332lb # [ 54.397248] niks3-server[1035]: 2026/09/22 10:36:51 INFO Registered completed upload object_key=nar/1wplyx8g7zja504cjvqmxis1q6laml9bidgiq4ya0i9p2pvqb9qp.nar.zst4333worker2 # [ 54.428999] nix-grpc-daemon[879]: time=2026-09-22T10:36:51.073Z level=INFO msg="Uploading 1 narinfos"4334lb # [ 54.410528] niks3-server[1035]: 2026/09/22 10:36:51 INFO Received sign narinfos request method=POST path=/api/pending_closures/6/sign4335lb # [ 54.419226] niks3-server[1035]: 2026/09/22 10:36:51 INFO Signed narinfos id=6 count=14336lb # [ 54.424312] niks3-server[1035]: 2026/09/22 10:36:51 INFO Registered completed upload object_key=nblg066ygaz2avzlmix3rnngr62hld8f.ls4337lb # [ 54.432268] niks3-server[1035]: 2026/09/22 10:36:51 INFO Received complete upload request method=POST path=/api/pending_closures/6/complete4338worker2 # [ 54.456722] nix-grpc-daemon[879]: time=2026-09-22T10:36:51.101Z level=INFO msg="Upload complete. (296ms)"4339lb # [ 54.439090] niks3-server[1035]: 2026/09/22 10:36:51 INFO Completed upload id=64340lb # [ 54.442511] niks3-server[1035]: 2026/09/22 10:36:51 INFO Registered completed upload object_key=nblg066ygaz2avzlmix3rnngr62hld8f.narinfo4341worker2 # [ 54.467195] nix-grpc-daemon[777]: ts=2026-09-22T10:36:51Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=1 drv=czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv assign_id=3 outputs=14342worker2 # [ 54.483349] nix-daemon[864]: reaped child process 976, status = succeeded4343client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 >&2, in 6.99 seconds)4344client: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 outPath4345client: (finished: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 outPath, in 0.23 seconds)4346client: must fail: test -e /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4347client: (finished: must fail: test -e /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top, in 0.02 seconds)4348client: must succeed: nix copy --from http://lb:5751 --no-check-sigs /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top && grep farm-top-t1 /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4349client # copying 3 paths...4350client # copying path '/nix/store/dg404zlvxfrspnxl6cc1qkc6i83m5a3a-farm-a' from 'http://lb:5751'...4351client # copying path '/nix/store/8n9g36vw9957r150a7wg1k6yj9fbqm8r-farm-b' from 'http://lb:5751'...4352client # copying path '/nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top' from 'http://lb:5751'...4353client: (finished: must succeed: nix copy --from http://lb:5751 --no-check-sigs /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top && grep farm-top-t1 /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top, in 0.28 seconds)4354(finished: subtest: DAG build is scheduled across workers and lands in the cache, in 7.63 seconds)4355subtest: repeat is answered Cached by the scheduler without building4356worker1: must succeed: nix-store --delete /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4357worker1 # finding garbage collector roots...4358worker1 # deleting specified paths...4359worker1 # deleting '/nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top'4360worker1 # deleting unused links...4361worker1 # note: hard linking is currently saving 0.0 KiB4362worker1: (finished: must succeed: nix-store --delete /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top, in 0.29 seconds)4363worker2: must succeed: nix-store --delete /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4364worker2 # finding garbage collector roots...4365worker2 # deleting specified paths...4366worker2 # deleting '/nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top'4367worker2 # deleting unused links...4368worker2 # note: hard linking is currently saving 0.0 KiB4369worker2: (finished: must succeed: nix-store --delete /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top, in 0.27 seconds)4370client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 >&24371worker2 # [ 55.996555] nix-grpc-daemon[777]: ts=2026-09-22T10:36:52Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:411444372client # this path will be fetched (0.0 KiB download, 0.2 KiB unpacked):4373client # /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4374worker2 # [ 56.014211] nix-grpc-daemon[777]: ts=2026-09-22T10:36:52Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444375worker2 # [ 56.025643] nix-grpc-daemon[879]: time=2026-09-22T10:36:52.669Z level=INFO msg="All 2 paths already cached"4376worker2 # [ 56.043801] nix-grpc-daemon[777]: ts=2026-09-22T10:36:52Z level=debug event=want client=2 drv=czd6ljynjaw3bl3w8iz1nx68whkdis8f-farm-top.drv cached=14377client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 >&2, in 0.36 seconds)4378client: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 outPath4379client: (finished: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 outPath, in 0.21 seconds)4380lb: must succeed: curl -sf localhost:9901/clusters4381lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)4382lb: must succeed: curl -sf localhost:9901/clusters4383lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)4384worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics4385worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)4386(finished: subtest: repeat is answered Cached by the scheduler without building, in 1.45 seconds)4387subtest: inputs already in the cache are not sent to the scheduler4388lb: must succeed: curl -sf localhost:9901/clusters4389lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.07 seconds)4390lb: must succeed: curl -sf localhost:9901/clusters4391lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)4392worker2: must succeed: journalctl -u nix-grpc-daemon -o cat | grep -c 'event=want ' || true4393worker2: (finished: must succeed: journalctl -u nix-grpc-daemon -o cat | grep -c 'event=want ' || true, in 0.06 seconds)4394client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 --argstr top t1b >&24395client # don't know how to build these paths:4396client # /nix/store/blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv4397worker2 # [ 57.068687] nix-grpc-daemon[777]: ts=2026-09-22T10:36:53Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:411444398worker2 # [ 57.075189] nix-grpc-daemon[777]: ts=2026-09-22T10:36:53Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444399worker2 # [ 57.086600] nix-grpc-daemon[879]: time=2026-09-22T10:36:53.730Z level=INFO msg="All 2 paths already cached"4400worker2 # [ 57.102876] nix-grpc-daemon[777]: ts=2026-09-22T10:36:53Z level=debug event=want client=3 drv=blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv cached=04401worker2 # [ 57.109846] nix-grpc-daemon[777]: ts=2026-09-22T10:36:53Z level=debug event=assigned drv=blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv worker=192.168.1.4:50051 clients=14402worker2 # [ 57.149950] nix-grpc-daemon[777]: ts=2026-09-22T10:36:53Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444403worker2 # [ 57.165678] nix-grpc-daemon[777]: ts=2026-09-22T10:36:53Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444404worker2 # [ 57.180664] nix-grpc-daemon[879]: time=2026-09-22T10:36:53.824Z level=INFO msg="All 2 paths already cached"4405client # copying 1 paths to 'lb:50051'...4406client # copying path '/nix/store/blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv' to 'lb:50051'...4407worker2 # [ 57.191783] nix-grpc-daemon[777]: ts=2026-09-22T10:36:53Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:411444408worker2 # [ 57.207355] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4409worker2 # [ 57.413778] nix-daemon[864]: accepted connection from pid 1026, user nix-grpc-daemon (trusted)4410worker2 # [ 57.434283] nix-daemon[864]: reaped child process 1029, status = succeeded4411worker2 # [ 57.604655] nix-daemon[864]: accepted connection from pid 1031, user nix-grpc-daemon (trusted)4412worker2 # [ 57.620877] nix-daemon[864]: reaped child process 1034, status = succeeded4413lb # [ 57.615642] niks3-server[1035]: 2026/09/22 10:36:54 INFO Received uploads request method=POST path=/api/pending_closures4414worker2 # [ 57.644588] nix-grpc-daemon[879]: time=2026-09-22T10:36:54.288Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"4415worker2 # [ 57.646027] nix-grpc-daemon[879]: time=2026-09-22T10:36:54.288Z level=INFO msg="Uploading blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv (752B)"4416lb # [ 57.645981] niks3-server[1035]: 2026/09/22 10:36:54 INFO Registered completed upload object_key=nar/1vy3ds565pni9mnkldcs6alimlb6bg39cimm60s4swkk5yjhwlqf.nar.zst4417worker2 # [ 57.677534] nix-grpc-daemon[879]: time=2026-09-22T10:36:54.321Z level=INFO msg="Uploading 1 narinfos"4418lb # [ 57.659508] niks3-server[1035]: 2026/09/22 10:36:54 INFO Received sign narinfos request method=POST path=/api/pending_closures/7/sign4419lb # [ 57.669297] niks3-server[1035]: 2026/09/22 10:36:54 INFO Signed narinfos id=7 count=14420lb # [ 57.673206] niks3-server[1035]: 2026/09/22 10:36:54 INFO Registered completed upload object_key=blgxcsidls7na1m5da3m0bja1prb87vl.ls4421lb # [ 57.681747] niks3-server[1035]: 2026/09/22 10:36:54 INFO Received complete upload request method=POST path=/api/pending_closures/7/complete4422worker2 # [ 57.704690] nix-grpc-daemon[879]: time=2026-09-22T10:36:54.349Z level=INFO msg="Upload complete. (480ms)"4423lb # [ 57.687057] niks3-server[1035]: 2026/09/22 10:36:54 INFO Completed upload id=74424lb # [ 57.689505] niks3-server[1035]: 2026/09/22 10:36:54 INFO Registered completed upload object_key=blgxcsidls7na1m5da3m0bja1prb87vl.narinfo4425worker2 # [ 57.712213] nix-grpc-daemon[777]: ts=2026-09-22T10:36:54Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_in=7524426worker2 # [ 57.722662] nix-grpc-daemon[777]: ts=2026-09-22T10:36:54Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444427worker2 # [ 57.727885] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4428worker2 # [ 57.732489] nix-daemon[864]: reaped child process 1024, status = succeeded4429client # building '/nix/store/blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv' on lb:50051...4430client # farm-top> worker2: building blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv4431worker2 # [ 57.745068] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4432client # farm-top> disabling binary cache 'https://cache.nixos.org' for 60 seconds4433client # farm-top> LOG-farm-top-t1b4434worker2 # [ 58.572304] nix-daemon[864]: reaped child process 1038, status = succeeded4435worker2 # [ 58.737272] nix-daemon[864]: accepted connection from pid 1054, user nix-grpc-daemon (trusted)4436worker2 # [ 58.757585] nix-daemon[864]: reaped child process 1057, status = succeeded4437lb # [ 58.751629] niks3-server[1035]: 2026/09/22 10:36:55 INFO Received uploads request method=POST path=/api/pending_closures4438worker2 # [ 58.780390] nix-grpc-daemon[879]: time=2026-09-22T10:36:55.423Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"4439worker2 # [ 58.781879] nix-grpc-daemon[879]: time=2026-09-22T10:36:55.423Z level=INFO msg="Uploading 0xnysbbdbmapgf23wb2dbsfhgh18yxvp-farm-top (232B)"4440lb # [ 58.787116] niks3-server[1035]: 2026/09/22 10:36:55 INFO Registered completed upload object_key=log/blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv4441lb # [ 58.801545] niks3-server[1035]: 2026/09/22 10:36:55 INFO Registered completed upload object_key=nar/1f6pjd04v1xmbxqqjhczkkfk4ns9qg6ssnspdalp41q9hkxf9yfr.nar.zst4442lb # [ 58.812998] niks3-server[1035]: 2026/09/22 10:36:55 INFO Received sign narinfos request method=POST path=/api/pending_closures/8/sign4443worker2 # [ 58.838160] nix-grpc-daemon[879]: time=2026-09-22T10:36:55.482Z level=INFO msg="Uploading 1 narinfos"4444lb # [ 58.820081] niks3-server[1035]: 2026/09/22 10:36:55 INFO Signed narinfos id=8 count=14445lb # [ 58.823758] niks3-server[1035]: 2026/09/22 10:36:55 INFO Registered completed upload object_key=0xnysbbdbmapgf23wb2dbsfhgh18yxvp.ls4446lb # [ 58.835290] niks3-server[1035]: 2026/09/22 10:36:55 INFO Received complete upload request method=POST path=/api/pending_closures/8/complete4447worker2 # [ 58.860525] nix-grpc-daemon[879]: time=2026-09-22T10:36:55.504Z level=INFO msg="Upload complete. (306ms)"4448lb # [ 58.843128] niks3-server[1035]: 2026/09/22 10:36:55 INFO Completed upload id=84449lb # [ 58.845986] niks3-server[1035]: 2026/09/22 10:36:55 INFO Registered completed upload object_key=0xnysbbdbmapgf23wb2dbsfhgh18yxvp.narinfo4450worker2 # [ 58.871146] nix-grpc-daemon[777]: ts=2026-09-22T10:36:55Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=1 drv=blgxcsidls7na1m5da3m0bja1prb87vl-farm-top.drv assign_id=4 outputs=14451worker2 # [ 58.887322] nix-daemon[864]: reaped child process 1036, status = succeeded4452client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 --argstr top t1b >&2, in 2.10 seconds)4453client: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 --argstr top t1b outPath4454client: (finished: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag t1 --argstr top t1b outPath, in 0.23 seconds)4455lb: must succeed: curl -sf localhost:9901/clusters4456lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)4457lb: must succeed: curl -sf localhost:9901/clusters4458lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)4459worker2: must succeed: journalctl -u nix-grpc-daemon -o cat | grep -c 'event=want ' || true4460worker2: (finished: must succeed: journalctl -u nix-grpc-daemon -o cat | grep -c 'event=want ' || true, in 0.07 seconds)4461(finished: subtest: inputs already in the cache are not sent to the scheduler, in 2.73 seconds)4462subtest: outputs read back through any worker carry the cache signature4463client: must succeed: nix path-info --sigs --store 'grpc://worker1:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&ca-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ca.pem' /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4464worker1 # [ 59.555482] nix-grpc-daemon[777]: ts=2026-09-22T10:36:56Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv6:%5B2001:db8:1::1%5D:605184465client # these 3 paths will be fetched (0.0 KiB download, 0.5 KiB unpacked):4466client # /nix/store/dg404zlvxfrspnxl6cc1qkc6i83m5a3a-farm-a4467client # /nix/store/8n9g36vw9957r150a7wg1k6yj9fbqm8r-farm-b4468client # /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4469worker1 # [ 59.595907] nix-grpc-daemon[777]: ts=2026-09-22T10:36:56Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv6:%5B2001:db8:1::1%5D:605184470worker1 # [ 59.606529] nix-grpc-daemon[777]: copying path '/nix/store/dg404zlvxfrspnxl6cc1qkc6i83m5a3a-farm-a' from 'http://lb:5751'...4471worker1 # [ 59.630917] nix-grpc-daemon[777]: copying path '/nix/store/8n9g36vw9957r150a7wg1k6yj9fbqm8r-farm-b' from 'http://lb:5751'...4472worker1 # [ 59.646003] nix-grpc-daemon[777]: copying path '/nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top' from 'http://lb:5751'...4473client: (finished: must succeed: nix path-info --sigs --store 'grpc://worker1:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&ca-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ca.pem' /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top, in 0.38 seconds)4474client: must succeed: nix path-info --sigs --store 'grpc://worker2:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&ca-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ca.pem' /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4475lb # [ 59.806554] envoy[896]: conn peer=[2001:db8:1::1]:32832 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=9549 tx=4438 ms=67374476lb # [ 59.812496] envoy[896]: conn peer=[2001:db8:1::1]:36044 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=993 tx=480 ms=694477lb # [ 59.817979] envoy[896]: conn peer=[2001:db8:1::1]:36048 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=4172 tx=1921 ms=18184478worker2 # [ 59.965582] nix-grpc-daemon[777]: ts=2026-09-22T10:36:56Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv6:%5B2001:db8:1::1%5D:504144479client # this path will be fetched (0.0 KiB download, 0.2 KiB unpacked):4480client # /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4481worker2 # [ 60.009127] nix-grpc-daemon[777]: ts=2026-09-22T10:36:56Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv6:%5B2001:db8:1::1%5D:504144482client: (finished: must succeed: nix path-info --sigs --store 'grpc://worker2:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&ca-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ca.pem' /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top, in 0.31 seconds)4483client: must succeed: nix-store --delete /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top && nix copy --from 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --option trusted-public-keys 'farm-test-1:RkClDwvfixdOwourBI4UD9hudE3xfU5EBQcMFUVuRV8=' /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4484client # finding garbage collector roots...4485client # removing stale temporary roots file "/nix/var/nix/temproots/857"4486client # removing stale temporary roots file "/nix/var/nix/temproots/816"4487client # removing stale temporary roots file "/nix/var/nix/temproots/837"4488client # removing stale temporary roots file "/nix/var/nix/temproots/849"4489client # removing stale temporary roots file "/nix/var/nix/temproots/801"4490client # removing stale temporary roots file "/nix/var/nix/temproots/870"4491client # deleting specified paths...4492client # deleting '/nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top'4493client # deleting unused links...4494client # note: hard linking is currently saving 0.0 KiB4495worker2 # [ 60.488919] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:411444496client # this path will be fetched (0.0 KiB download, 0.2 KiB unpacked):4497client # /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top4498worker2 # [ 60.500494] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444499worker2 # [ 60.509262] nix-grpc-daemon[777]: copying path '/nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top' from 'http://lb:5751'...4500worker2 # [ 60.534153] nix-grpc-daemon[879]: time=2026-09-22T10:36:57.177Z level=INFO msg="All 1 paths already cached"4501worker2 # [ 60.542737] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:411444502worker2 # [ 60.551945] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:411444503client # copying 1 paths...4504worker2 # [ 60.568672] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:411444505client # copying path '/nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top' from 'grpc://lb:50051'...4506worker2 # [ 60.595868] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:411444507worker2 # [ 60.602555] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=info event=rpc method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_out=2324508client: (finished: must succeed: nix-store --delete /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top && nix copy --from 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --option trusted-public-keys 'farm-test-1:RkClDwvfixdOwourBI4UD9hudE3xfU5EBQcMFUVuRV8=' /nix/store/nblg066ygaz2avzlmix3rnngr62hld8f-farm-top, in 0.65 seconds)4509(finished: subtest: outputs read back through any worker carry the cache signature, in 1.35 seconds)4510subtest: two clients wanting the same drv build it once4511client: must succeed: systemd-run --unit dedup1 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag dedup4512client # [ 60.719426] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag dedup.4513client # Running as unit: dedup1.service; invocation ID: 760dc159baf14757bb18e5ef3915e1394514client: (finished: must succeed: systemd-run --unit dedup1 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag dedup, in 0.08 seconds)4515client: must succeed: sleep 1; systemd-run --unit dedup2 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag dedup4516worker2 # [ 61.011341] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:411444517client # [ 60.978478] nix[921]: don't know how to build these paths:4518client # [ 60.979426] nix[921]: /nix/store/cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv4519worker2 # [ 61.028142] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=want client=4 drv=cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv cached=04520worker2 # [ 61.034308] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=assigned drv=cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv worker=192.168.1.3:50051 clients=14521worker1 # [ 61.047216] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684522worker1 # [ 61.065816] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:330684523client # [ 61.075081] nix[921]: copying 1 paths to 'lb:50051'...4524worker1 # [ 61.080822] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:330684525client # [ 61.077886] nix[921]: copying path '/nix/store/cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv' to 'lb:50051'...4526worker1 # [ 61.096130] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4527worker1 # [ 61.122489] nix-grpc-daemon[777]: ts=2026-09-22T10:36:57Z level=info event=niks3_push_started pid=8734528worker1 # [ 61.389833] nix-grpc-daemon[873]: time=2026-09-22T10:36:58.019Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.key ca=""4529worker1 # [ 61.577577] nix-daemon[844]: accepted connection from pid 886, user nix-grpc-daemon (trusted)4530worker1 # [ 61.595535] nix-daemon[844]: reaped child process 889, status = succeeded4531worker1 # [ 61.765516] nix-daemon[844]: accepted connection from pid 891, user nix-grpc-daemon (trusted)4532worker1 # [ 61.783885] nix-daemon[844]: reaped child process 894, status = succeeded4533lb # [ 61.813115] niks3-server[1035]: 2026/09/22 10:36:58 INFO Received uploads request method=POST path=/api/pending_closures4534worker1 # [ 61.807402] nix-grpc-daemon[873]: time=2026-09-22T10:36:58.438Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4535worker1 # [ 61.810582] nix-grpc-daemon[873]: time=2026-09-22T10:36:58.440Z level=INFO msg="Uploading cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv (432B)"4536lb # [ 61.851680] niks3-server[1035]: 2026/09/22 10:36:58 INFO Registered completed upload object_key=nar/0l7vpaziqxpw0srrsgimqvaa0dyxa9glicn3ilkwk06l64hn837w.nar.zst4537client # [ 61.823705] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag dedup.4538lb # [ 61.864731] niks3-server[1035]: 2026/09/22 10:36:58 INFO Received sign narinfos request method=POST path=/api/pending_closures/9/sign4539worker1 # [ 61.855006] nix-grpc-daemon[873]: time=2026-09-22T10:36:58.485Z level=INFO msg="Uploading 1 narinfos"4540client: (finished: must succeed: sleep 1; systemd-run --unit dedup2 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag dedup, in 1.12 seconds)4541lb # [ 61.878783] niks3-server[1035]: 2026/09/22 10:36:58 INFO Signed narinfos id=9 count=14542lb # [ 61.885700] niks3-server[1035]: 2026/09/22 10:36:58 INFO Registered completed upload object_key=cp23830c44gs2dw4xy80s5wqyjm7vw84.ls4543lb # [ 61.895401] niks3-server[1035]: 2026/09/22 10:36:58 INFO Received complete upload request method=POST path=/api/pending_closures/9/complete4544worker1 # [ 61.887921] nix-grpc-daemon[873]: time=2026-09-22T10:36:58.518Z level=INFO msg="Upload complete. (490ms)"4545lb # [ 61.904942] niks3-server[1035]: 2026/09/22 10:36:58 INFO Completed upload id=94546lb # [ 61.909478] niks3-server[1035]: 2026/09/22 10:36:58 INFO Registered completed upload object_key=cp23830c44gs2dw4xy80s5wqyjm7vw84.narinfo4547worker1 # [ 61.895444] nix-grpc-daemon[777]: ts=2026-09-22T10:36:58Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_in=4324548worker1 # [ 61.907129] nix-grpc-daemon[777]: ts=2026-09-22T10:36:58Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684549worker1 # [ 61.912184] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4550worker1 # [ 61.918243] nix-daemon[844]: reaped child process 871, status = succeeded4551client # Running as unit: dedup2.service; invocation ID: 75e678aae8ed4d9c8bef63197ef55a6d4552client # [ 61.921180] nix[921]: building '/nix/store/cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv' on lb:50051...4553worker1 # [ 61.929682] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4554worker2 # [ 62.216804] nix-grpc-daemon[777]: ts=2026-09-22T10:36:58Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:411444555client # [ 62.183663] nix[937]: don't know how to build these paths:4556client # [ 62.186086] nix[937]: /nix/store/cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv4557worker2 # [ 62.234817] nix-grpc-daemon[777]: ts=2026-09-22T10:36:58Z level=debug event=want client=5 drv=cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv cached=04558worker1 # [ 62.207066] nix-grpc-daemon[777]: ts=2026-09-22T10:36:58Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684559client # [ 62.208131] nix[937]: building '/nix/store/cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv' on lb:50051...4560client: must succeed: sleep 24561client: (finished: must succeed: sleep 2, in 2.07 seconds)4562client: waiting for success: ! systemctl is-active dedup1 dedup24563worker1 # [ 66.772467] nix-daemon[844]: reaped child process 902, status = succeeded4564worker1 # [ 66.957636] nix-daemon[844]: accepted connection from pid 932, user nix-grpc-daemon (trusted)4565worker1 # [ 66.974470] nix-daemon[844]: reaped child process 935, status = succeeded4566lb # [ 67.005291] niks3-server[1035]: 2026/09/22 10:37:03 INFO Received uploads request method=POST path=/api/pending_closures4567worker1 # [ 66.999126] nix-grpc-daemon[873]: time=2026-09-22T10:37:03.628Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4568worker1 # [ 67.001054] nix-grpc-daemon[873]: time=2026-09-22T10:37:03.628Z level=INFO msg="Uploading vamz940my4dma9a2daklr9pw9x9pp9hp-slow-dedup (120B)"4569lb # [ 67.048206] niks3-server[1035]: 2026/09/22 10:37:03 INFO Registered completed upload object_key=log/cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv4570lb # [ 67.060789] niks3-server[1035]: 2026/09/22 10:37:03 INFO Registered completed upload object_key=nar/02xhr0h0sj1ji8m0s8995xzbbb8mhbqdpdl1ggx94yqqaykihvz7.nar.zst4571lb # [ 67.071950] niks3-server[1035]: 2026/09/22 10:37:03 INFO Received sign narinfos request method=POST path=/api/pending_closures/10/sign4572worker1 # [ 67.063041] nix-grpc-daemon[873]: time=2026-09-22T10:37:03.692Z level=INFO msg="Uploading 1 narinfos"4573lb # [ 67.079644] niks3-server[1035]: 2026/09/22 10:37:03 INFO Signed narinfos id=10 count=14574lb # [ 67.086041] niks3-server[1035]: 2026/09/22 10:37:03 INFO Registered completed upload object_key=vamz940my4dma9a2daklr9pw9x9pp9hp.ls4575lb # [ 67.098790] niks3-server[1035]: 2026/09/22 10:37:03 INFO Received complete upload request method=POST path=/api/pending_closures/10/complete4576lb # [ 67.107814] niks3-server[1035]: 2026/09/22 10:37:03 INFO Completed upload id=104577worker1 # [ 67.091086] nix-grpc-daemon[873]: time=2026-09-22T10:37:03.722Z level=INFO msg="Upload complete. (341ms)"4578lb # [ 67.111302] niks3-server[1035]: 2026/09/22 10:37:03 INFO Registered completed upload object_key=vamz940my4dma9a2daklr9pw9x9pp9hp.narinfo4579worker1 # [ 67.105059] nix-grpc-daemon[777]: ts=2026-09-22T10:37:03Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=4 drv=cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv assign_id=5 outputs=14580worker1 # [ 67.120659] nix-grpc-daemon[777]: ts=2026-09-22T10:37:03Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=5 drv=cp23830c44gs2dw4xy80s5wqyjm7vw84-slow-dedup.drv assign_id=5 outputs=14581worker1 # [ 67.127189] nix-daemon[844]: reaped child process 900, status = succeeded4582client # [ 67.142720] systemd[1]: dedup2.service: Deactivated successfully.4583client # [ 67.150405] systemd[1]: dedup2.service: Consumed 211ms CPU time over 5.325s wall clock time, 33.2M memory peak, 6K incoming IP traffic, 5.6K outgoing IP traffic.4584client # [ 67.195832] systemd[1]: dedup1.service: Deactivated successfully.4585client # [ 67.200985] systemd[1]: dedup1.service: Consumed 165ms CPU time over 6.479s wall clock time, 41.2M memory peak, 7.4K incoming IP traffic, 8.6K outgoing IP traffic.4586client: (finished: waiting for success: ! systemctl is-active dedup1 dedup2, in 1.14 seconds)4587client: must succeed: systemctl show -p Result dedup1 dedup2 | grep -c success | grep -qx 2 || { journalctl -u dedup1 -u dedup2 >&2; false; }4588client: (finished: must succeed: systemctl show -p Result dedup1 dedup2 | grep -c success | grep -qx 2 || { journalctl -u dedup1 -u dedup2 >&2; false; }, in 0.07 seconds)4589worker1: must succeed: journalctl -u nix-grpc-daemon -o cat | grep 'method=BuildDerivation.*slow-dedup' | grep -o 'assign_id=[0-9]*' || true4590worker1: (finished: must succeed: journalctl -u nix-grpc-daemon -o cat | grep 'method=BuildDerivation.*slow-dedup' | grep -o 'assign_id=[0-9]*' || true, in 0.07 seconds)4591worker2: must succeed: journalctl -u nix-grpc-daemon -o cat | grep 'method=BuildDerivation.*slow-dedup' | grep -o 'assign_id=[0-9]*' || true4592worker2: (finished: must succeed: journalctl -u nix-grpc-daemon -o cat | grep 'method=BuildDerivation.*slow-dedup' | grep -o 'assign_id=[0-9]*' || true, in 0.07 seconds)4593worker1: must succeed: curl -sf http://127.0.0.1:9464/metrics4594worker1: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.07 seconds)4595worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics4596worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)4597(finished: subtest: two clients wanting the same drv build it once, in 7.46 seconds)4598subtest: the first client leaving does not fail the second4599client: must succeed: systemd-run --unit share1 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag share4600client # [ 68.189889] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag share.4601client: (finished: must succeed: systemd-run --unit share1 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag share, in 0.09 seconds)4602client: must succeed: sleep 1; systemd-run --unit share2 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag share4603worker1 # [ 68.457287] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:330684604client # Running as unit: share1.service; invocation ID: 514d110f92d144e5b1ef86a9c6c1da1a4605client # [ 68.459550] nix[977]: don't know how to build these paths:4606client # [ 68.461449] nix[977]: /nix/store/rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv4607worker2 # [ 68.516931] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=debug event=want client=6 drv=rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv cached=04608worker2 # [ 68.522467] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=debug event=assigned drv=rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv worker=192.168.1.4:50051 clients=14609worker2 # [ 68.562711] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444610worker2 # [ 68.579844] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444611client # [ 68.552228] nix[977]: copying 1 paths to 'lb:50051'...4612worker2 # [ 68.594980] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:411444613client # [ 68.561517] nix[977]: copying path '/nix/store/rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv' to 'lb:50051'...4614worker2 # [ 68.618326] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4615worker2 # [ 68.833144] nix-daemon[864]: accepted connection from pid 1098, user nix-grpc-daemon (trusted)4616worker2 # [ 68.851080] nix-daemon[864]: reaped child process 1101, status = succeeded4617worker2 # [ 69.026517] nix-daemon[864]: accepted connection from pid 1103, user nix-grpc-daemon (trusted)4618worker2 # [ 69.043815] nix-daemon[864]: reaped child process 1106, status = succeeded4619lb # [ 69.040836] niks3-server[1035]: 2026/09/22 10:37:05 INFO Received uploads request method=POST path=/api/pending_closures4620worker2 # [ 69.068985] nix-grpc-daemon[879]: time=2026-09-22T10:37:05.712Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4621worker2 # [ 69.070466] nix-grpc-daemon[879]: time=2026-09-22T10:37:05.712Z level=INFO msg="Uploading rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv (432B)"4622lb # [ 69.072571] niks3-server[1035]: 2026/09/22 10:37:05 INFO Registered completed upload object_key=nar/0chdjpc9an2ckdblh08lnyrmzvw3h8qs76cn5ydwjwvgwkrm731l.nar.zst4623lb # [ 69.088266] niks3-server[1035]: 2026/09/22 10:37:05 INFO Received sign narinfos request method=POST path=/api/pending_closures/11/sign4624worker2 # [ 69.112912] nix-grpc-daemon[879]: time=2026-09-22T10:37:05.756Z level=INFO msg="Uploading 1 narinfos"4625lb # [ 69.100203] niks3-server[1035]: 2026/09/22 10:37:05 INFO Signed narinfos id=11 count=14626lb # [ 69.107221] niks3-server[1035]: 2026/09/22 10:37:05 INFO Registered completed upload object_key=rxafkggp1xca5h7p0r6gwpq3qqp2kn63.ls4627lb # [ 69.115204] niks3-server[1035]: 2026/09/22 10:37:05 INFO Received complete upload request method=POST path=/api/pending_closures/11/complete4628worker2 # [ 69.139012] nix-grpc-daemon[879]: time=2026-09-22T10:37:05.783Z level=INFO msg="Upload complete. (502ms)"4629lb # [ 69.123454] niks3-server[1035]: 2026/09/22 10:37:05 INFO Completed upload id=114630lb # [ 69.127184] niks3-server[1035]: 2026/09/22 10:37:05 INFO Registered completed upload object_key=rxafkggp1xca5h7p0r6gwpq3qqp2kn63.narinfo4631worker2 # [ 69.148272] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_in=4324632worker2 # [ 69.163182] nix-grpc-daemon[777]: ts=2026-09-22T10:37:05Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444633worker2 # [ 69.170531] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4634worker2 # [ 69.176086] nix-daemon[864]: reaped child process 1096, status = succeeded4635client # [ 69.145178] nix[977]: building '/nix/store/rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv' on lb:50051...4636worker2 # [ 69.188220] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4637client # [ 69.301838] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag share.4638client # Running as unit: share2.service; invocation ID: c07fc4fc03e04cf384dc8c475af9396b4639client: (finished: must succeed: sleep 1; systemd-run --unit share2 nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag share, in 1.12 seconds)4640client # [ 69.576992] nix[993]: don't know how to build these paths:4641client # [ 69.577922] nix[993]: /nix/store/rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv4642worker1 # [ 69.580888] nix-grpc-daemon[777]: ts=2026-09-22T10:37:06Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:330684643worker2 # [ 69.625314] nix-grpc-daemon[777]: ts=2026-09-22T10:37:06Z level=debug event=want client=7 drv=rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv cached=04644client # [ 69.589778] nix[993]: building '/nix/store/rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv' on lb:50051...4645worker2 # [ 69.630199] nix-grpc-daemon[777]: ts=2026-09-22T10:37:06Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444646lb # [ 69.804456] envoy[896]: conn peer=[2001:db8:1::1]:36058 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=969 tx=2183 ms=1804647lb # [ 69.809658] envoy[896]: conn peer=[2001:db8:1::1]:36064 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=410 tx=411 ms=774648lb # [ 69.809790] envoy[896]: conn peer=[2001:db8:1::1]:44726 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=1430 tx=1194 ms=49384649lb # [ 69.809972] envoy[896]: conn peer=[2001:db8:1::1]:44722 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=3035 tx=1579 ms=61834650client: must succeed: sleep 2; systemctl kill -s INT share14651client # [ 72.731837] systemd[1]: share1.service: Sent signal SIGINT to main process 977 (nix) on client request.4652client # [ 72.742056] nix[977]: shutting downrxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv: error: gRPC BuildDerivation on 'lb:50051' failed: Stream removed (CANCELLED)4653client: (finished: must succeed: sleep 2; systemctl kill -s INT share1, in 2.10 seconds)4654client: waiting for success: ! systemctl is-active share14655client # [ 72.760555] nix[977]: error: interrupted by the user4656client # [ 72.809407] systemd[1]: share1.service: Main process exited, code=exited, status=1/FAILURE4657client # [ 72.814171] systemd[1]: share1.service: Failed with result 'exit-code'.4658client # [ 72.816213] systemd[1]: share1.service: Consumed 175ms CPU time over 4.622s wall clock time, 41.4M memory peak, 6.2K incoming IP traffic, 8K outgoing IP traffic.4659client: (finished: waiting for success: ! systemctl is-active share1, in 0.08 seconds)4660client: waiting for success: ! systemctl is-active share24661worker2 # [ 73.214429] nix-daemon[864]: reaped child process 1110, status = succeeded4662worker2 # [ 73.383424] nix-daemon[864]: accepted connection from pid 1146, user nix-grpc-daemon (trusted)4663worker2 # [ 73.405307] nix-daemon[864]: reaped child process 1149, status = succeeded4664lb # [ 73.397553] niks3-server[1035]: 2026/09/22 10:37:10 INFO Received uploads request method=POST path=/api/pending_closures4665worker2 # [ 73.430330] nix-grpc-daemon[879]: time=2026-09-22T10:37:10.072Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4666worker2 # [ 73.432240] nix-grpc-daemon[879]: time=2026-09-22T10:37:10.073Z level=INFO msg="Uploading fjww0fd4rwk1s35jav3cyal62l24n356-slow-share (120B)"4667lb # [ 73.443250] niks3-server[1035]: 2026/09/22 10:37:10 INFO Registered completed upload object_key=log/rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv4668lb # [ 73.455418] niks3-server[1035]: 2026/09/22 10:37:10 INFO Registered completed upload object_key=nar/172qq270pqbv9ipsik19pmld0m7fj8r5nyyxq3slfx9n7zcaa9s0.nar.zst4669worker2 # [ 73.483021] nix-grpc-daemon[879]: time=2026-09-22T10:37:10.127Z level=INFO msg="Uploading 1 narinfos"4670lb # [ 73.467809] niks3-server[1035]: 2026/09/22 10:37:10 INFO Received sign narinfos request method=POST path=/api/pending_closures/12/sign4671lb # [ 73.474600] niks3-server[1035]: 2026/09/22 10:37:10 INFO Signed narinfos id=12 count=14672lb # [ 73.480667] niks3-server[1035]: 2026/09/22 10:37:10 INFO Registered completed upload object_key=fjww0fd4rwk1s35jav3cyal62l24n356.ls4673lb # [ 73.483816] niks3-server[1035]: 2026/09/22 10:37:10 INFO Received complete upload request method=POST path=/api/pending_closures/12/complete4674lb # [ 73.487853] niks3-server[1035]: 2026/09/22 10:37:10 INFO Completed upload id=124675worker2 # [ 73.505553] nix-grpc-daemon[879]: time=2026-09-22T10:37:10.149Z level=INFO msg="Upload complete. (303ms)"4676lb # [ 73.490318] niks3-server[1035]: 2026/09/22 10:37:10 INFO Registered completed upload object_key=fjww0fd4rwk1s35jav3cyal62l24n356.narinfo4677worker2 # [ 73.517036] nix-grpc-daemon[777]: ts=2026-09-22T10:37:10Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=3 drv=rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv assign_id=6 outputs=14678worker2 # [ 73.532344] nix-grpc-daemon[777]: ts=2026-09-22T10:37:10Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=4 drv=rxafkggp1xca5h7p0r6gwpq3qqp2kn63-slow-share.drv assign_id=6 outputs=14679worker2 # [ 73.538781] nix-daemon[864]: reaped child process 1108, status = succeeded4680client # [ 73.562536] systemd[1]: share2.service: Deactivated successfully.4681client # [ 73.567393] systemd[1]: share2.service: Consumed 173ms CPU time over 4.265s wall clock time, 33.2M memory peak, 6K incoming IP traffic, 5.7K outgoing IP traffic.4682client: (finished: waiting for success: ! systemctl is-active share2, in 1.15 seconds)4683client: must succeed: systemctl show -p Result share2 | grep -q success || { journalctl -u share2 >&2; false; }4684client: (finished: must succeed: systemctl show -p Result share2 | grep -q success || { journalctl -u share2 >&2; false; }, in 0.07 seconds)4685(finished: subtest: the first client leaving does not fail the second, in 6.24 seconds)4686subtest: build hook: nix-daemon with builders = grpc://lb4687client: must succeed: NIX_REMOTE=daemon nix build --log-format internal-json --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --print-out-paths --no-link -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag hook 2>/tmp/hook.log4688client # [ 74.568407] systemd[1]: Started Nix Daemon.4689client # [ 74.775959] nix-daemon[1040]: accepted connection from pid 1037, user root (trusted)4690worker1 # [ 75.014483] nix-grpc-daemon[777]: ts=2026-09-22T10:37:11Z level=debug event=rpc_start method=StoreInfo cn=ci-1 peer=ipv4:192.168.1.2:330684691worker1 # [ 75.022254] nix-grpc-daemon[777]: ts=2026-09-22T10:37:11Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:330684692worker2 # [ 75.074585] nix-grpc-daemon[777]: ts=2026-09-22T10:37:11Z level=debug event=want client=8 drv=xs7jpxrx1nnx7zbhs6p2v76pjq3964kv-farm-a.drv cached=04693worker2 # [ 75.080944] nix-grpc-daemon[777]: ts=2026-09-22T10:37:11Z level=debug event=assigned drv=xs7jpxrx1nnx7zbhs6p2v76pjq3964kv-farm-a.drv worker=192.168.1.3:50051 clients=14694worker1 # [ 75.084232] nix-grpc-daemon[777]: ts=2026-09-22T10:37:11Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684695worker1 # [ 75.101599] nix-grpc-daemon[777]: ts=2026-09-22T10:37:11Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:330684696worker1 # [ 75.119348] nix-grpc-daemon[777]: ts=2026-09-22T10:37:11Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:330684697worker1 # [ 75.136672] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4698worker1 # [ 75.369235] nix-daemon[844]: accepted connection from pid 964, user nix-grpc-daemon (trusted)4699worker1 # [ 75.389176] nix-daemon[844]: reaped child process 967, status = succeeded4700worker1 # [ 75.587216] nix-daemon[844]: accepted connection from pid 969, user nix-grpc-daemon (trusted)4701worker1 # [ 75.605586] nix-daemon[844]: reaped child process 972, status = succeeded4702lb # [ 75.638254] niks3-server[1035]: 2026/09/22 10:37:12 INFO Received uploads request method=POST path=/api/pending_closures4703worker1 # [ 75.633240] nix-grpc-daemon[873]: time=2026-09-22T10:37:12.262Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4704worker1 # [ 75.634762] nix-grpc-daemon[873]: time=2026-09-22T10:37:12.262Z level=INFO msg="Uploading xs7jpxrx1nnx7zbhs6p2v76pjq3964kv-farm-a.drv (512B)"4705lb # [ 75.672706] niks3-server[1035]: 2026/09/22 10:37:12 INFO Registered completed upload object_key=nar/1f1i5kab3qhx7pg1k0ma5r1m7ipfazkfkfb944dm7qcjr466pwij.nar.zst4706lb # [ 75.690224] niks3-server[1035]: 2026/09/22 10:37:12 INFO Received sign narinfos request method=POST path=/api/pending_closures/13/sign4707worker1 # [ 75.679012] nix-grpc-daemon[873]: time=2026-09-22T10:37:12.309Z level=INFO msg="Uploading 1 narinfos"4708lb # [ 75.700594] niks3-server[1035]: 2026/09/22 10:37:12 INFO Signed narinfos id=13 count=14709lb # [ 75.705698] niks3-server[1035]: 2026/09/22 10:37:12 INFO Registered completed upload object_key=xs7jpxrx1nnx7zbhs6p2v76pjq3964kv.ls4710lb # [ 75.714537] niks3-server[1035]: 2026/09/22 10:37:12 INFO Received complete upload request method=POST path=/api/pending_closures/13/complete4711worker1 # [ 75.701653] nix-grpc-daemon[873]: time=2026-09-22T10:37:12.332Z level=INFO msg="Upload complete. (544ms)"4712lb # [ 75.720102] niks3-server[1035]: 2026/09/22 10:37:12 INFO Completed upload id=134713lb # [ 75.722717] niks3-server[1035]: 2026/09/22 10:37:12 INFO Registered completed upload object_key=xs7jpxrx1nnx7zbhs6p2v76pjq3964kv.narinfo4714worker1 # [ 75.712316] nix-grpc-daemon[777]: ts=2026-09-22T10:37:12Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_in=5124715worker1 # [ 75.720380] nix-grpc-daemon[777]: ts=2026-09-22T10:37:12Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684716worker1 # [ 75.735614] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4717worker1 # [ 75.738197] nix-daemon[844]: reaped child process 962, status = succeeded4718worker1 # [ 75.752391] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4719worker1 # [ 77.190042] nix-daemon[844]: reaped child process 976, status = succeeded4720worker1 # [ 77.359443] nix-daemon[844]: accepted connection from pid 992, user nix-grpc-daemon (trusted)4721worker1 # [ 77.374573] nix-daemon[844]: reaped child process 995, status = succeeded4722lb # [ 77.409801] niks3-server[1035]: 2026/09/22 10:37:14 INFO Received uploads request method=POST path=/api/pending_closures4723worker1 # [ 77.404881] nix-grpc-daemon[873]: time=2026-09-22T10:37:14.035Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4724worker1 # [ 77.406343] nix-grpc-daemon[873]: time=2026-09-22T10:37:14.035Z level=INFO msg="Uploading yfs64sxmsg77fnzczdvc3g3yfdwh9fks-farm-a (128B)"4725lb # [ 77.452781] niks3-server[1035]: 2026/09/22 10:37:14 INFO Registered completed upload object_key=log/xs7jpxrx1nnx7zbhs6p2v76pjq3964kv-farm-a.drv4726lb # [ 77.463066] niks3-server[1035]: 2026/09/22 10:37:14 INFO Registered completed upload object_key=nar/0pbbxl4kxy7zl7x4lp43ywljwn82mxzna2wgpb1bz1z765nwp9ki.nar.zst4727lb # [ 77.473796] niks3-server[1035]: 2026/09/22 10:37:14 INFO Received sign narinfos request method=POST path=/api/pending_closures/14/sign4728worker1 # [ 77.462944] nix-grpc-daemon[873]: time=2026-09-22T10:37:14.093Z level=INFO msg="Uploading 1 narinfos"4729lb # [ 77.479859] niks3-server[1035]: 2026/09/22 10:37:14 INFO Signed narinfos id=14 count=14730lb # [ 77.484903] niks3-server[1035]: 2026/09/22 10:37:14 INFO Registered completed upload object_key=yfs64sxmsg77fnzczdvc3g3yfdwh9fks.ls4731lb # [ 77.496597] niks3-server[1035]: 2026/09/22 10:37:14 INFO Received complete upload request method=POST path=/api/pending_closures/14/complete4732worker1 # [ 77.486242] nix-grpc-daemon[873]: time=2026-09-22T10:37:14.117Z level=INFO msg="Upload complete. (320ms)"4733lb # [ 77.505387] niks3-server[1035]: 2026/09/22 10:37:14 INFO Completed upload id=144734lb # [ 77.509436] niks3-server[1035]: 2026/09/22 10:37:14 INFO Registered completed upload object_key=yfs64sxmsg77fnzczdvc3g3yfdwh9fks.narinfo4735worker1 # [ 77.497104] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=1 drv=xs7jpxrx1nnx7zbhs6p2v76pjq3964kv-farm-a.drv assign_id=7 outputs=14736worker1 # [ 77.516606] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:330684737worker1 # [ 77.521212] nix-daemon[844]: reaped child process 974, status = succeeded4738worker1 # [ 77.532219] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=rpc_start method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:330684739worker1 # [ 77.539714] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=info event=rpc method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_out=1284740worker2 # [ 77.894819] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=rpc_start method=StoreInfo cn=ci-1 peer=ipv4:192.168.1.2:411444741worker2 # [ 77.903369] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444742worker2 # [ 77.927391] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=want client=9 drv=r1jsr4b2g1qwly395z6idx3z2362ih8i-farm-b.drv cached=04743worker2 # [ 77.933645] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=assigned drv=r1jsr4b2g1qwly395z6idx3z2362ih8i-farm-b.drv worker=192.168.1.3:50051 clients=14744worker1 # [ 77.901786] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684745worker1 # [ 77.916673] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:330684746worker1 # [ 77.930118] nix-grpc-daemon[873]: time=2026-09-22T10:37:14.561Z level=INFO msg="All 1 paths already cached"4747worker1 # [ 77.940819] nix-grpc-daemon[777]: ts=2026-09-22T10:37:14Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:330684748worker1 # [ 77.954788] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4749worker1 # [ 78.171926] nix-daemon[844]: accepted connection from pid 999, user nix-grpc-daemon (trusted)4750worker1 # [ 78.188156] nix-daemon[844]: reaped child process 1002, status = succeeded4751worker1 # [ 78.362648] nix-daemon[844]: accepted connection from pid 1004, user nix-grpc-daemon (trusted)4752worker1 # [ 78.378677] nix-daemon[844]: reaped child process 1007, status = succeeded4753lb # [ 78.407213] niks3-server[1035]: 2026/09/22 10:37:15 INFO Received uploads request method=POST path=/api/pending_closures4754worker1 # [ 78.399548] nix-grpc-daemon[873]: time=2026-09-22T10:37:15.030Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"4755worker1 # [ 78.401174] nix-grpc-daemon[873]: time=2026-09-22T10:37:15.030Z level=INFO msg="Uploading r1jsr4b2g1qwly395z6idx3z2362ih8i-farm-b.drv (624B)"4756lb # [ 78.437556] niks3-server[1035]: 2026/09/22 10:37:15 INFO Registered completed upload object_key=nar/15k66v3mh3v6p5vymv4pv3v3sfl56i2b1ia2rf093c37w6bhc04m.nar.zst4757worker1 # [ 78.439138] nix-grpc-daemon[873]: time=2026-09-22T10:37:15.069Z level=INFO msg="Uploading 1 narinfos"4758lb # [ 78.448692] niks3-server[1035]: 2026/09/22 10:37:15 INFO Received sign narinfos request method=POST path=/api/pending_closures/15/sign4759lb # [ 78.461751] niks3-server[1035]: 2026/09/22 10:37:15 INFO Signed narinfos id=15 count=14760lb # [ 78.468720] niks3-server[1035]: 2026/09/22 10:37:15 INFO Registered completed upload object_key=r1jsr4b2g1qwly395z6idx3z2362ih8i.ls4761lb # [ 78.475534] niks3-server[1035]: 2026/09/22 10:37:15 INFO Received complete upload request method=POST path=/api/pending_closures/15/complete4762worker1 # [ 78.464380] nix-grpc-daemon[873]: time=2026-09-22T10:37:15.095Z level=INFO msg="Upload complete. (492ms)"4763lb # [ 78.481789] niks3-server[1035]: 2026/09/22 10:37:15 INFO Completed upload id=154764lb # [ 78.484421] niks3-server[1035]: 2026/09/22 10:37:15 INFO Registered completed upload object_key=r1jsr4b2g1qwly395z6idx3z2362ih8i.narinfo4765worker1 # [ 78.472131] nix-grpc-daemon[777]: ts=2026-09-22T10:37:15Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_in=6244766worker1 # [ 78.484052] nix-grpc-daemon[777]: ts=2026-09-22T10:37:15Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684767worker1 # [ 78.490167] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4768worker1 # [ 78.494974] nix-daemon[844]: reaped child process 997, status = succeeded4769worker1 # [ 78.507861] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4770lb # [ 79.805362] envoy[896]: conn peer=[2001:db8:1::1]:44738 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=2964 tx=790 ms=43174771lb # [ 79.810377] envoy[896]: conn peer=[2001:db8:1::1]:44748 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=1443 tx=1194 ms=39574772lb # [ 79.816204] envoy[896]: conn peer=[2001:db8:1::1]:57382 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=3414 tx=1963 ms=25874773lb # [ 79.820968] envoy[896]: conn peer=[2001:db8:1::1]:57384 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=410 tx=333 ms=714774worker1 # [ 79.942602] nix-daemon[844]: reaped child process 1011, status = succeeded4775worker1 # [ 80.136336] nix-daemon[844]: accepted connection from pid 1027, user nix-grpc-daemon (trusted)4776worker1 # [ 80.155229] nix-daemon[844]: reaped child process 1030, status = succeeded4777lb # [ 80.189281] niks3-server[1035]: 2026/09/22 10:37:16 INFO Received uploads request method=POST path=/api/pending_closures4778worker1 # [ 80.183213] nix-grpc-daemon[873]: time=2026-09-22T10:37:16.812Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"4779worker1 # [ 80.184827] nix-grpc-daemon[873]: time=2026-09-22T10:37:16.812Z level=INFO msg="Uploading la6ryzsihag4zpxgxls54fsg34dvlalk-farm-b (176B)"4780lb # [ 80.220645] niks3-server[1035]: 2026/09/22 10:37:16 INFO Registered completed upload object_key=log/r1jsr4b2g1qwly395z6idx3z2362ih8i-farm-b.drv4781lb # [ 80.238062] niks3-server[1035]: 2026/09/22 10:37:16 INFO Registered completed upload object_key=nar/069d0xnyw2q345iqzx6agxc5fxifc6q36bkkklw2aa4qwanxdd1g.nar.zst4782lb # [ 80.249022] niks3-server[1035]: 2026/09/22 10:37:16 INFO Received sign narinfos request method=POST path=/api/pending_closures/16/sign4783worker1 # [ 80.239414] nix-grpc-daemon[873]: time=2026-09-22T10:37:16.870Z level=INFO msg="Uploading 1 narinfos"4784lb # [ 80.256153] niks3-server[1035]: 2026/09/22 10:37:16 INFO Signed narinfos id=16 count=14785lb # [ 80.264223] niks3-server[1035]: 2026/09/22 10:37:16 INFO Registered completed upload object_key=la6ryzsihag4zpxgxls54fsg34dvlalk.ls4786lb # [ 80.274530] niks3-server[1035]: 2026/09/22 10:37:16 INFO Received complete upload request method=POST path=/api/pending_closures/16/complete4787lb # [ 80.284598] niks3-server[1035]: 2026/09/22 10:37:16 INFO Completed upload id=164788worker1 # [ 80.266848] nix-grpc-daemon[873]: time=2026-09-22T10:37:16.897Z level=INFO msg="Upload complete. (348ms)"4789lb # [ 80.287797] niks3-server[1035]: 2026/09/22 10:37:16 INFO Registered completed upload object_key=la6ryzsihag4zpxgxls54fsg34dvlalk.narinfo4790worker1 # [ 80.281183] nix-grpc-daemon[777]: ts=2026-09-22T10:37:16Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=1 drv=r1jsr4b2g1qwly395z6idx3z2362ih8i-farm-b.drv assign_id=8 outputs=14791worker1 # [ 80.298348] nix-grpc-daemon[777]: ts=2026-09-22T10:37:16Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:330684792worker1 # [ 80.302795] nix-daemon[844]: reaped child process 1009, status = succeeded4793worker1 # [ 80.316133] nix-grpc-daemon[777]: ts=2026-09-22T10:37:16Z level=debug event=rpc_start method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:330684794worker1 # [ 80.321933] nix-grpc-daemon[777]: ts=2026-09-22T10:37:16Z level=info event=rpc method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_out=1764795worker1 # [ 80.646838] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=rpc_start method=StoreInfo cn=ci-1 peer=ipv4:192.168.1.2:330684796worker2 # [ 80.687498] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444797worker2 # [ 80.714627] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=want client=10 drv=aa4abikjj4w30bfx1p8hxmq0916qbz1c-farm-top.drv cached=04798worker1 # [ 80.683189] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684799worker2 # [ 80.721313] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=assigned drv=aa4abikjj4w30bfx1p8hxmq0916qbz1c-farm-top.drv worker=192.168.1.3:50051 clients=14800worker1 # [ 80.698785] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:330684801worker1 # [ 80.713120] nix-grpc-daemon[873]: time=2026-09-22T10:37:17.344Z level=INFO msg="All 2 paths already cached"4802worker1 # [ 80.723441] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:330684803worker1 # [ 80.737868] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4804worker1 # [ 80.941432] nix-daemon[844]: accepted connection from pid 1034, user nix-grpc-daemon (trusted)4805worker1 # [ 80.958579] nix-daemon[844]: reaped child process 1037, status = succeeded4806worker1 # [ 81.132437] nix-daemon[844]: accepted connection from pid 1039, user nix-grpc-daemon (trusted)4807worker1 # [ 81.150110] nix-daemon[844]: reaped child process 1042, status = succeeded4808lb # [ 81.180980] niks3-server[1035]: 2026/09/22 10:37:17 INFO Received uploads request method=POST path=/api/pending_closures4809worker1 # [ 81.175473] nix-grpc-daemon[873]: time=2026-09-22T10:37:17.806Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"4810worker1 # [ 81.177104] nix-grpc-daemon[873]: time=2026-09-22T10:37:17.806Z level=INFO msg="Uploading aa4abikjj4w30bfx1p8hxmq0916qbz1c-farm-top.drv (752B)"4811lb # [ 81.214448] niks3-server[1035]: 2026/09/22 10:37:17 INFO Registered completed upload object_key=nar/1ahq7sp1sl2scxqzfjd8gqimpqfkjbyi9ivxdhdwx0jzh2gbv74b.nar.zst4812worker1 # [ 81.212372] nix-grpc-daemon[873]: time=2026-09-22T10:37:17.842Z level=INFO msg="Uploading 1 narinfos"4813lb # [ 81.230521] niks3-server[1035]: 2026/09/22 10:37:17 INFO Received sign narinfos request method=POST path=/api/pending_closures/17/sign4814lb # [ 81.241370] niks3-server[1035]: 2026/09/22 10:37:17 INFO Signed narinfos id=17 count=14815lb # [ 81.243742] niks3-server[1035]: 2026/09/22 10:37:17 INFO Registered completed upload object_key=aa4abikjj4w30bfx1p8hxmq0916qbz1c.ls4816lb # [ 81.252530] niks3-server[1035]: 2026/09/22 10:37:17 INFO Received complete upload request method=POST path=/api/pending_closures/17/complete4817lb # [ 81.261475] niks3-server[1035]: 2026/09/22 10:37:17 INFO Completed upload id=174818worker1 # [ 81.244647] nix-grpc-daemon[873]: time=2026-09-22T10:37:17.875Z level=INFO msg="Upload complete. (489ms)"4819lb # [ 81.264623] niks3-server[1035]: 2026/09/22 10:37:17 INFO Registered completed upload object_key=aa4abikjj4w30bfx1p8hxmq0916qbz1c.narinfo4820worker1 # [ 81.252316] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_in=7524821worker1 # [ 81.267952] nix-grpc-daemon[777]: ts=2026-09-22T10:37:17Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684822worker1 # [ 81.275265] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4823worker1 # [ 81.280655] nix-daemon[844]: reaped child process 1032, status = succeeded4824worker1 # [ 81.295324] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4825worker1 # [ 82.123874] nix-daemon[844]: reaped child process 1046, status = succeeded4826worker1 # [ 82.289761] nix-daemon[844]: accepted connection from pid 1062, user nix-grpc-daemon (trusted)4827worker1 # [ 82.308572] nix-daemon[844]: reaped child process 1065, status = succeeded4828lb # [ 82.341771] niks3-server[1035]: 2026/09/22 10:37:18 INFO Received uploads request method=POST path=/api/pending_closures4829worker1 # [ 82.335004] nix-grpc-daemon[873]: time=2026-09-22T10:37:18.964Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"4830worker1 # [ 82.337451] nix-grpc-daemon[873]: time=2026-09-22T10:37:18.964Z level=INFO msg="Uploading 39qxwzy2zkij70y9zvbr59pb0df38w5h-farm-top (232B)"4831lb # [ 82.379130] niks3-server[1035]: 2026/09/22 10:37:19 INFO Registered completed upload object_key=log/aa4abikjj4w30bfx1p8hxmq0916qbz1c-farm-top.drv4832lb # [ 82.393429] niks3-server[1035]: 2026/09/22 10:37:19 INFO Registered completed upload object_key=nar/0vq3rhfhhbqyx5pbvaa9ddj9wbw9arph334y3bp3idfrikwnjc92.nar.zst4833lb # [ 82.402686] niks3-server[1035]: 2026/09/22 10:37:19 INFO Received sign narinfos request method=POST path=/api/pending_closures/18/sign4834worker1 # [ 82.393370] nix-grpc-daemon[873]: time=2026-09-22T10:37:19.023Z level=INFO msg="Uploading 1 narinfos"4835lb # [ 82.411538] niks3-server[1035]: 2026/09/22 10:37:19 INFO Signed narinfos id=18 count=14836lb # [ 82.418188] niks3-server[1035]: 2026/09/22 10:37:19 INFO Registered completed upload object_key=39qxwzy2zkij70y9zvbr59pb0df38w5h.ls4837lb # [ 82.428101] niks3-server[1035]: 2026/09/22 10:37:19 INFO Received complete upload request method=POST path=/api/pending_closures/18/complete4838lb # [ 82.434018] niks3-server[1035]: 2026/09/22 10:37:19 INFO Completed upload id=184839worker1 # [ 82.417268] nix-grpc-daemon[873]: time=2026-09-22T10:37:19.048Z level=INFO msg="Upload complete. (312ms)"4840lb # [ 82.437092] niks3-server[1035]: 2026/09/22 10:37:19 INFO Registered completed upload object_key=39qxwzy2zkij70y9zvbr59pb0df38w5h.narinfo4841worker2 # [ 82.469246] nix-grpc-daemon[777]: ts=2026-09-22T10:37:19Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:411444842worker1 # [ 82.429203] nix-grpc-daemon[777]: ts=2026-09-22T10:37:19Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=1 drv=aa4abikjj4w30bfx1p8hxmq0916qbz1c-farm-top.drv assign_id=9 outputs=14843worker1 # [ 82.446157] nix-daemon[844]: reaped child process 1044, status = succeeded4844worker2 # [ 82.482927] nix-grpc-daemon[777]: copying path '/nix/store/yfs64sxmsg77fnzczdvc3g3yfdwh9fks-farm-a' from 'http://lb:5751'...4845worker2 # [ 82.498272] nix-grpc-daemon[777]: copying path '/nix/store/la6ryzsihag4zpxgxls54fsg34dvlalk-farm-b' from 'http://lb:5751'...4846worker2 # [ 82.513017] nix-grpc-daemon[777]: copying path '/nix/store/39qxwzy2zkij70y9zvbr59pb0df38w5h-farm-top' from 'http://lb:5751'...4847worker1 # [ 82.518843] nix-grpc-daemon[777]: ts=2026-09-22T10:37:19Z level=debug event=rpc_start method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:330684848worker1 # [ 82.524370] nix-grpc-daemon[777]: ts=2026-09-22T10:37:19Z level=info event=rpc method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_out=2324849client # [ 82.606243] nix-daemon[1040]: reaped child process 1042, status = succeeded4850client: (finished: must succeed: NIX_REMOTE=daemon nix build --log-format internal-json --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --print-out-paths --no-link -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag hook 2>/tmp/hook.log, in 8.26 seconds)4851client: must succeed: grep farm-top-hook /nix/store/39qxwzy2zkij70y9zvbr59pb0df38w5h-farm-top4852client: (finished: must succeed: grep farm-top-hook /nix/store/39qxwzy2zkij70y9zvbr59pb0df38w5h-farm-top, in 0.04 seconds)4853client: must succeed: grep -F 'LOG-farm-top-hook' /tmp/hook.log | grep -qF '"type":101' && grep -F 'farmPhase' /tmp/hook.log | grep -qF '"type":104' || { cat /tmp/hook.log >&2; false; }4854client: (finished: must succeed: grep -F 'LOG-farm-top-hook' /tmp/hook.log | grep -qF '"type":101' && grep -F 'farmPhase' /tmp/hook.log | grep -qF '"type":104' || { cat /tmp/hook.log >&2; false; }, in 0.05 seconds)4855client: must succeed: grep -F '"type":105' /tmp/hook.log | grep -F lb:50051 | grep -qF '"fields":["/nix/store/' || { grep -F '"type":105' /tmp/hook.log >&2; false; }4856client: (finished: must succeed: grep -F '"type":105' /tmp/hook.log | grep -F lb:50051 | grep -qF '"fields":["/nix/store/' || { grep -F '"type":105' /tmp/hook.log >&2; false; }, in 0.04 seconds)4857(finished: subtest: build hook: nix-daemon with builders = grpc://lb, in 8.40 seconds)4858subtest: an input only one worker has still reaches the builder4859client: must succeed: nix-build --no-out-link /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix -A input --argstr tag w1only4860client # this derivation will be built:4861client # /nix/store/4llc9br53475v1z1bdqrjyfi2sjvwnvr-local-input-w1only.drv4862client # building '/nix/store/4llc9br53475v1z1bdqrjyfi2sjvwnvr-local-input-w1only.drv'...4863client: (finished: must succeed: nix-build --no-out-link /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix -A input --argstr tag w1only, in 0.44 seconds)4864client: must succeed: nix-store --export /nix/store/dymj0d1vj8q0ns6p096qk7fii766dap6-local-input-w1only > /tmp/shared/inp.closure4865client: (finished: must succeed: nix-store --export /nix/store/dymj0d1vj8q0ns6p096qk7fii766dap6-local-input-w1only > /tmp/shared/inp.closure, in 0.20 seconds)4866worker1: must succeed: nix-store --import < /tmp/shared/inp.closure4867worker1: (finished: must succeed: nix-store --import < /tmp/shared/inp.closure, in 0.24 seconds)4868worker2: must fail: test -e /nix/store/dymj0d1vj8q0ns6p096qk7fii766dap6-local-input-w1only4869worker2: (finished: must fail: test -e /nix/store/dymj0d1vj8q0ns6p096qk7fii766dap6-local-input-w1only, in 0.04 seconds)4870client: must succeed: NIX_REMOTE=daemon nix build -L --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --no-link -f /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix job --argstr tag w1only --argstr salt a >&24871client # [ 83.885094] nix-daemon[1040]: accepted connection from pid 1123, user root (trusted)4872client # this derivation will be built:4873client # /nix/store/71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv4874client # building '/nix/store/71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv' on 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux'...4875worker2 # [ 84.140890] nix-grpc-daemon[777]: ts=2026-09-22T10:37:20Z level=debug event=rpc_start method=StoreInfo cn=ci-1 peer=ipv4:192.168.1.2:411444876worker2 # [ 84.148645] nix-grpc-daemon[777]: ts=2026-09-22T10:37:20Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444877client # copying 1 paths...4878client # copying path '/nix/store/dymj0d1vj8q0ns6p096qk7fii766dap6-local-input-w1only' to 'grpc://lb:50051'...4879worker1 # [ 84.126023] nix-grpc-daemon[777]: ts=2026-09-22T10:37:20Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:330684880worker1 # [ 84.141499] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4881worker1 # [ 84.354333] nix-daemon[844]: accepted connection from pid 1074, user nix-grpc-daemon (trusted)4882worker1 # [ 84.371023] nix-daemon[844]: reaped child process 1077, status = succeeded4883lb # [ 84.400719] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received uploads request method=POST path=/api/pending_closures4884worker1 # [ 84.394717] nix-grpc-daemon[873]: time=2026-09-22T10:37:21.025Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"4885worker1 # [ 84.396285] nix-grpc-daemon[873]: time=2026-09-22T10:37:21.025Z level=INFO msg="Uploading dymj0d1vj8q0ns6p096qk7fii766dap6-local-input-w1only (120B)"4886lb # [ 84.430536] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=nar/08ghl87f009pwqz85nbqv9hm0ln6gai6ggzilfmw3qnkaalfi6ss.nar.zst4887worker1 # [ 84.426374] nix-grpc-daemon[873]: time=2026-09-22T10:37:21.057Z level=INFO msg="Uploading 1 narinfos"4888lb # [ 84.446042] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received sign narinfos request method=POST path=/api/pending_closures/19/sign4889lb # [ 84.457477] niks3-server[1035]: 2026/09/22 10:37:21 INFO Signed narinfos id=19 count=14890worker1 # [ 84.452725] nix-grpc-daemon[873]: time=2026-09-22T10:37:21.083Z level=INFO msg="Upload complete. (296ms)"4891lb # [ 84.469556] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=dymj0d1vj8q0ns6p096qk7fii766dap6.ls4892lb # [ 84.474005] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received complete upload request method=POST path=/api/pending_closures/19/complete4893lb # [ 84.478704] niks3-server[1035]: 2026/09/22 10:37:21 INFO Completed upload id=194894lb # [ 84.481523] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=dymj0d1vj8q0ns6p096qk7fii766dap6.narinfo4895worker1 # [ 84.464280] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_in=1204896worker1 # [ 84.476110] nix-daemon[844]: reaped child process 1072, status = succeeded4897worker2 # [ 84.514034] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=debug event=want client=11 drv=71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv cached=04898worker2 # [ 84.521765] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=debug event=assigned drv=71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv worker=192.168.1.4:50051 clients=14899worker2 # [ 84.558267] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444900worker2 # [ 84.573049] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444901client # copying 2 paths to 'lb:50051'...4902client # copying path '/nix/store/4llc9br53475v1z1bdqrjyfi2sjvwnvr-local-input-w1only.drv' to 'lb:50051'...4903worker2 # [ 84.590242] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:411444904client # copying path '/nix/store/71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv' to 'lb:50051'...4905worker2 # [ 84.605463] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4906worker2 # [ 84.814065] nix-daemon[864]: accepted connection from pid 1167, user nix-grpc-daemon (trusted)4907worker2 # [ 84.833223] nix-daemon[864]: reaped child process 1170, status = succeeded4908worker2 # [ 85.003379] nix-daemon[864]: accepted connection from pid 1172, user nix-grpc-daemon (trusted)4909worker2 # [ 85.019915] nix-daemon[864]: reaped child process 1175, status = succeeded4910lb # [ 85.013637] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received uploads request method=POST path=/api/pending_closures4911lb # [ 85.023485] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received uploads request method=POST path=/api/pending_closures4912worker2 # [ 85.052397] nix-grpc-daemon[879]: time=2026-09-22T10:37:21.695Z level=INFO msg="Uploading 2 paths to lb (0 already cached)"4913worker2 # [ 85.053840] nix-grpc-daemon[879]: time=2026-09-22T10:37:21.695Z level=INFO msg="Uploading 71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv (568B)"4914worker2 # [ 85.055562] nix-grpc-daemon[879]: time=2026-09-22T10:37:21.695Z level=INFO msg="Uploading 4llc9br53475v1z1bdqrjyfi2sjvwnvr-local-input-w1only.drv (416B)"4915lb # [ 85.060995] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=nar/0p6zgq1gdngshl62q0hv3lslqwv90s7fmflwc7wgnycf4ka6ik69.nar.zst4916lb # [ 85.073722] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=nar/0i2vbprwp0wxa61nkss61gzivqmmjh6klkgv0rlfnxk46rcaba1v.nar.zst4917lb # [ 85.086539] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received sign narinfos request method=POST path=/api/pending_closures/20/sign4918worker2 # [ 85.111012] nix-grpc-daemon[879]: time=2026-09-22T10:37:21.754Z level=INFO msg="Uploading 3 narinfos"4919lb # [ 85.094992] niks3-server[1035]: 2026/09/22 10:37:21 INFO Signed narinfos id=20 count=14920lb # [ 85.099053] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received sign narinfos request method=POST path=/api/pending_closures/21/sign4921lb # [ 85.105516] niks3-server[1035]: 2026/09/22 10:37:21 INFO Signed narinfos id=21 count=24922lb # [ 85.108801] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=4llc9br53475v1z1bdqrjyfi2sjvwnvr.ls4923lb # [ 85.114099] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=71l2x9mybp0i90r6pdnglwjnayj4xpl8.ls4924lb # [ 85.125388] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received complete upload request method=POST path=/api/pending_closures/20/complete4925lb # [ 85.135556] niks3-server[1035]: 2026/09/22 10:37:21 INFO Completed upload id=204926worker2 # [ 85.153635] nix-grpc-daemon[879]: time=2026-09-22T10:37:21.797Z level=INFO msg="Upload complete. (527ms)"4927lb # [ 85.141045] niks3-server[1035]: 2026/09/22 10:37:21 INFO Received complete upload request method=POST path=/api/pending_closures/21/complete4928lb # [ 85.147261] niks3-server[1035]: 2026/09/22 10:37:21 INFO Completed upload id=214929lb # [ 85.149627] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=71l2x9mybp0i90r6pdnglwjnayj4xpl8.narinfo4930lb # [ 85.152210] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=4llc9br53475v1z1bdqrjyfi2sjvwnvr.narinfo4931worker2 # [ 85.166957] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=2 nar_bytes_in=9844932lb # [ 85.154658] niks3-server[1035]: 2026/09/22 10:37:21 INFO Registered completed upload object_key=4llc9br53475v1z1bdqrjyfi2sjvwnvr.narinfo4933worker2 # [ 85.184612] nix-grpc-daemon[777]: ts=2026-09-22T10:37:21Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411444934worker2 # [ 85.193380] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4935worker2 # [ 85.199761] nix-daemon[864]: reaped child process 1165, status = succeeded4936worker2 # [ 85.214530] nix-grpc-daemon[777]: copying path '/nix/store/dymj0d1vj8q0ns6p096qk7fii766dap6-local-input-w1only' from 'http://lb:5751'...4937client # building '/nix/store/71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv' on lb:50051...4938worker2 # [ 85.229859] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4939client # uses-input-w1onlya> worker2: building 71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv4940client # uses-input-w1onlya> disabling binary cache 'https://cache.nixos.org' for 60 seconds4941worker2 # [ 86.067605] nix-daemon[864]: reaped child process 1180, status = succeeded4942worker2 # [ 86.230303] nix-daemon[864]: accepted connection from pid 1196, user nix-grpc-daemon (trusted)4943worker2 # [ 86.248867] nix-daemon[864]: reaped child process 1199, status = succeeded4944lb # [ 86.245957] niks3-server[1035]: 2026/09/22 10:37:22 INFO Received uploads request method=POST path=/api/pending_closures4945worker2 # [ 86.276382] nix-grpc-daemon[879]: time=2026-09-22T10:37:22.918Z level=INFO msg="Uploading 0 paths to lb (1 already cached)"4946lb # [ 86.276622] niks3-server[1035]: 2026/09/22 10:37:22 INFO Received sign narinfos request method=POST path=/api/pending_closures/22/sign4947worker2 # [ 86.306444] nix-grpc-daemon[879]: time=2026-09-22T10:37:22.950Z level=INFO msg="Uploading 1 narinfos"4948lb # [ 86.291119] niks3-server[1035]: 2026/09/22 10:37:22 INFO Signed narinfos id=22 count=14949lb # [ 86.299291] niks3-server[1035]: 2026/09/22 10:37:22 INFO Registered completed upload object_key=2i7aa0cdpp60vpdm180r56gqfh25m7hc.ls4950lb # [ 86.306247] niks3-server[1035]: 2026/09/22 10:37:22 INFO Registered completed upload object_key=log/71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv4951lb # [ 86.315810] niks3-server[1035]: 2026/09/22 10:37:22 INFO Received complete upload request method=POST path=/api/pending_closures/22/complete4952lb # [ 86.322059] niks3-server[1035]: 2026/09/22 10:37:22 INFO Completed upload id=224953worker2 # [ 86.340407] nix-grpc-daemon[879]: time=2026-09-22T10:37:22.984Z level=INFO msg="Upload complete. (295ms)"4954lb # [ 86.324593] niks3-server[1035]: 2026/09/22 10:37:22 INFO Registered completed upload object_key=2i7aa0cdpp60vpdm180r56gqfh25m7hc.narinfo4955worker2 # [ 86.348753] nix-grpc-daemon[777]: ts=2026-09-22T10:37:22Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=1 drv=71l2x9mybp0i90r6pdnglwjnayj4xpl8-uses-input-w1onlya.drv assign_id=10 outputs=14956worker2 # [ 86.362099] nix-daemon[864]: reaped child process 1177, status = succeeded4957client # copying 1 paths...4958worker1 # [ 86.361819] nix-grpc-daemon[777]: ts=2026-09-22T10:37:22Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:330684959worker1 # [ 86.374857] nix-grpc-daemon[777]: copying path '/nix/store/2i7aa0cdpp60vpdm180r56gqfh25m7hc-uses-input-w1onlya' from 'http://lb:5751'...4960client # copying path '/nix/store/2i7aa0cdpp60vpdm180r56gqfh25m7hc-uses-input-w1onlya' from 'grpc://lb:50051'...4961worker1 # [ 86.422921] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=rpc_start method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:330684962worker1 # [ 86.428227] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=info event=rpc method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_out=1204963client # [ 86.542501] nix-daemon[1040]: reaped child process 1126, status = succeeded4964client: (finished: must succeed: NIX_REMOTE=daemon nix build -L --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --no-link -f /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix job --argstr tag w1only --argstr salt a >&2, in 2.89 seconds)4965client: must succeed: NIX_REMOTE=daemon nix build -L --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --no-link -f /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix job --argstr tag w1only --argstr salt b >&24966client # [ 86.757306] nix-daemon[1040]: accepted connection from pid 1145, user root (trusted)4967client # this derivation will be built:4968client # /nix/store/12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv4969worker1 # [ 86.976139] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=rpc_start method=StoreInfo cn=ci-1 peer=ipv4:192.168.1.2:330684970client # building '/nix/store/12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv' on 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux'...4971worker2 # [ 87.014604] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411444972worker2 # [ 87.024934] nix-grpc-daemon[879]: time=2026-09-22T10:37:23.669Z level=INFO msg="All 1 paths already cached"4973worker2 # [ 87.041865] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=want client=12 drv=12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv cached=04974worker1 # [ 87.009717] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330684975worker2 # [ 87.049789] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=assigned drv=12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv worker=192.168.1.3:50051 clients=14976worker1 # [ 87.024744] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:330684977client # copying 1 paths to 'lb:50051'...4978client # copying path '/nix/store/12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv' to 'lb:50051'...4979worker1 # [ 87.044114] nix-grpc-daemon[777]: ts=2026-09-22T10:37:23Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:330684980worker1 # [ 87.059353] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)4981worker1 # [ 87.082594] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 33 ms (attempt 1/5)4982worker1 # [ 87.122378] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 138 ms (attempt 2/5)4983worker1 # [ 87.267631] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 235 ms (attempt 3/5)4984worker1 # [ 87.511554] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 610 ms (attempt 4/5)4985worker1 # [ 88.129541] nix-grpc-daemon[777]: disabling binary cache 'https://cache.nixos.org' for 60 seconds4986worker1 # [ 88.137130] nix-grpc-daemon[777]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org4987worker1 # [ 88.144391] nix-grpc-daemon[777]: copying path '/nix/store/4llc9br53475v1z1bdqrjyfi2sjvwnvr-local-input-w1only.drv' from 'http://lb:5751'...4988worker1 # [ 88.375258] nix-daemon[844]: accepted connection from pid 1087, user nix-grpc-daemon (trusted)4989worker1 # [ 88.392818] nix-daemon[844]: reaped child process 1090, status = succeeded4990worker1 # [ 88.563843] nix-daemon[844]: accepted connection from pid 1092, user nix-grpc-daemon (trusted)4991worker1 # [ 88.578196] nix-daemon[844]: reaped child process 1095, status = succeeded4992lb # [ 88.611968] niks3-server[1035]: 2026/09/22 10:37:25 INFO Received uploads request method=POST path=/api/pending_closures4993worker1 # [ 88.604748] nix-grpc-daemon[873]: time=2026-09-22T10:37:25.235Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"4994worker1 # [ 88.606176] nix-grpc-daemon[873]: time=2026-09-22T10:37:25.235Z level=INFO msg="Uploading 12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv (568B)"4995lb # [ 88.642966] niks3-server[1035]: 2026/09/22 10:37:25 INFO Registered completed upload object_key=nar/14jznd749vfl63s1gv7i29r5z6jcai2byp2nz83dr5gib6z2cyaz.nar.zst4996worker1 # [ 88.636133] nix-grpc-daemon[873]: time=2026-09-22T10:37:25.266Z level=INFO msg="Uploading 1 narinfos"4997lb # [ 88.658432] niks3-server[1035]: 2026/09/22 10:37:25 INFO Received sign narinfos request method=POST path=/api/pending_closures/23/sign4998lb # [ 88.669775] niks3-server[1035]: 2026/09/22 10:37:25 INFO Signed narinfos id=23 count=14999lb # [ 88.676478] niks3-server[1035]: 2026/09/22 10:37:25 INFO Registered completed upload object_key=12ialcyr3l4m98rzfgk6apzimy55z3j8.ls5000worker1 # [ 88.665034] nix-grpc-daemon[873]: time=2026-09-22T10:37:25.295Z level=INFO msg="Upload complete. (499ms)"5001lb # [ 88.679999] niks3-server[1035]: 2026/09/22 10:37:25 INFO Received complete upload request method=POST path=/api/pending_closures/23/complete5002lb # [ 88.687172] niks3-server[1035]: 2026/09/22 10:37:25 INFO Completed upload id=235003worker1 # [ 88.672635] nix-grpc-daemon[777]: ts=2026-09-22T10:37:25Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=1 paths=1 nar_bytes_in=5685004lb # [ 88.691166] niks3-server[1035]: 2026/09/22 10:37:25 INFO Registered completed upload object_key=12ialcyr3l4m98rzfgk6apzimy55z3j8.narinfo5005worker1 # [ 88.685391] nix-grpc-daemon[777]: ts=2026-09-22T10:37:25Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:330685006worker1 # [ 88.690991] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)5007worker1 # [ 88.695086] nix-daemon[844]: reaped child process 1080, status = succeeded5008client # building '/nix/store/12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv' on lb:50051...5009worker1 # [ 88.706685] nix-daemon[844]: accepted connection from pid 777, user nix-grpc-daemon (trusted)5010client # uses-input-w1onlyb> node-a: building 12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv5011client # uses-input-w1onlyb> disabling binary cache 'https://cache.nixos.org' for 60 seconds5012lb # [ 89.806824] envoy[896]: conn peer=[2001:db8:1::1]:57410 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=436 tx=375 ms=705013lb # [ 89.811721] envoy[896]: conn peer=[2001:db8:1::1]:57394 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=3888 tx=2179 ms=25215014lb # [ 89.816676] envoy[896]: conn peer=[2001:db8:1::1]:57414 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=4352 tx=2390 ms=18885015lb # [ 89.821475] envoy[896]: conn peer=[2001:db8:1::1]:32820 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=412 tx=406 ms=235016lb # [ 89.827221] envoy[896]: conn peer=[2001:db8:1::1]:32836 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=4232 tx=2070 ms=23745017lb # [ 89.832112] envoy[896]: conn peer=[2001:db8:1::1]:32846 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=422 tx=328 ms=625018worker1 # [ 89.929496] nix-daemon[844]: reaped child process 1099, status = succeeded5019worker1 # [ 90.088619] nix-daemon[844]: accepted connection from pid 1115, user nix-grpc-daemon (trusted)5020worker1 # [ 90.107010] nix-daemon[844]: reaped child process 1118, status = succeeded5021lb # [ 90.136716] niks3-server[1035]: 2026/09/22 10:37:26 INFO Received uploads request method=POST path=/api/pending_closures5022worker1 # [ 90.131702] nix-grpc-daemon[873]: time=2026-09-22T10:37:26.759Z level=INFO msg="Uploading 0 paths to lb (1 already cached)"5023lb # [ 90.165041] niks3-server[1035]: 2026/09/22 10:37:26 INFO Registered completed upload object_key=jfy62nvxcd0b9rkhly152q21ddbrcn28.ls5024lb # [ 90.177267] niks3-server[1035]: 2026/09/22 10:37:26 INFO Received sign narinfos request method=POST path=/api/pending_closures/24/sign5025worker1 # [ 90.167399] nix-grpc-daemon[873]: time=2026-09-22T10:37:26.798Z level=INFO msg="Uploading 1 narinfos"5026lb # [ 90.190680] niks3-server[1035]: 2026/09/22 10:37:26 INFO Signed narinfos id=24 count=15027lb # [ 90.197135] niks3-server[1035]: 2026/09/22 10:37:26 INFO Registered completed upload object_key=log/12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv5028lb # [ 90.208117] niks3-server[1035]: 2026/09/22 10:37:26 INFO Received complete upload request method=POST path=/api/pending_closures/24/complete5029lb # [ 90.212796] niks3-server[1035]: 2026/09/22 10:37:26 INFO Completed upload id=245030worker1 # [ 90.195774] nix-grpc-daemon[873]: time=2026-09-22T10:37:26.826Z level=INFO msg="Upload complete. (280ms)"5031lb # [ 90.216464] niks3-server[1035]: 2026/09/22 10:37:26 INFO Registered completed upload object_key=jfy62nvxcd0b9rkhly152q21ddbrcn28.narinfo5032worker1 # [ 90.204715] nix-grpc-daemon[777]: ts=2026-09-22T10:37:26Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=1 drv=12ialcyr3l4m98rzfgk6apzimy55z3j8-uses-input-w1onlyb.drv assign_id=11 outputs=15033client # copying 1 paths...5034client # copying path '/nix/store/jfy62nvxcd0b9rkhly152q21ddbrcn28-uses-input-w1onlyb' from 'grpc://lb:50051'...5035worker1 # [ 90.217253] nix-grpc-daemon[777]: ts=2026-09-22T10:37:26Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:330685036worker1 # [ 90.220574] nix-daemon[844]: reaped child process 1097, status = succeeded5037worker1 # [ 90.240946] nix-grpc-daemon[777]: ts=2026-09-22T10:37:26Z level=debug event=rpc_start method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:330685038worker1 # [ 90.245392] nix-grpc-daemon[777]: ts=2026-09-22T10:37:26Z level=info event=rpc method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:33068 duration_s=0 paths=1 nar_bytes_out=1205039client # [ 90.366865] nix-daemon[1040]: reaped child process 1148, status = succeeded5040client: (finished: must succeed: NIX_REMOTE=daemon nix build -L --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --no-link -f /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix job --argstr tag w1only --argstr salt b >&2, in 3.83 seconds)5041client: must succeed: NIX_REMOTE=daemon nix build -L --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --no-link -f /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix job --argstr tag w1only --argstr salt c >&25042client # [ 90.588642] nix-daemon[1040]: accepted connection from pid 1167, user root (trusted)5043client # this derivation will be built:5044client # /nix/store/n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv5045client # building '/nix/store/n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv' on 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux'...5046worker2 # [ 90.842447] nix-grpc-daemon[777]: ts=2026-09-22T10:37:27Z level=debug event=rpc_start method=StoreInfo cn=ci-1 peer=ipv4:192.168.1.2:411445047worker2 # [ 90.849130] nix-grpc-daemon[777]: ts=2026-09-22T10:37:27Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411445048worker2 # [ 90.860634] nix-grpc-daemon[879]: time=2026-09-22T10:37:27.504Z level=INFO msg="All 1 paths already cached"5049worker2 # [ 90.877691] nix-grpc-daemon[777]: ts=2026-09-22T10:37:27Z level=debug event=want client=13 drv=n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv cached=05050worker2 # [ 90.885046] nix-grpc-daemon[777]: ts=2026-09-22T10:37:27Z level=debug event=assigned drv=n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv worker=192.168.1.4:50051 clients=15051worker2 # [ 90.922311] nix-grpc-daemon[777]: ts=2026-09-22T10:37:27Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411445052worker2 # [ 90.938417] nix-grpc-daemon[777]: ts=2026-09-22T10:37:27Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:411445053worker2 # [ 90.953516] nix-grpc-daemon[879]: time=2026-09-22T10:37:27.597Z level=INFO msg="All 1 paths already cached"5054client # copying 1 paths to 'lb:50051'...5055client # copying path '/nix/store/n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv' to 'lb:50051'...5056worker2 # [ 90.965081] nix-grpc-daemon[777]: ts=2026-09-22T10:37:27Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:411445057worker2 # [ 90.979824] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)5058worker2 # [ 91.196097] nix-daemon[864]: accepted connection from pid 1207, user nix-grpc-daemon (trusted)5059worker2 # [ 91.217123] nix-daemon[864]: reaped child process 1210, status = succeeded5060worker2 # [ 91.393107] nix-daemon[864]: accepted connection from pid 1212, user nix-grpc-daemon (trusted)5061worker2 # [ 91.411642] nix-daemon[864]: reaped child process 1215, status = succeeded5062lb # [ 91.406180] niks3-server[1035]: 2026/09/22 10:37:28 INFO Received uploads request method=POST path=/api/pending_closures5063worker2 # [ 91.433745] nix-grpc-daemon[879]: time=2026-09-22T10:37:28.078Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"5064worker2 # [ 91.435677] nix-grpc-daemon[879]: time=2026-09-22T10:37:28.079Z level=INFO msg="Uploading n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv (568B)"5065lb # [ 91.435573] niks3-server[1035]: 2026/09/22 10:37:28 INFO Registered completed upload object_key=nar/1qd3ps964395j2cc3rlqd58m21s7f6na9s0k2z5kclx0c3l4sl40.nar.zst5066lb # [ 91.450990] niks3-server[1035]: 2026/09/22 10:37:28 INFO Received sign narinfos request method=POST path=/api/pending_closures/25/sign5067worker2 # [ 91.475047] nix-grpc-daemon[879]: time=2026-09-22T10:37:28.119Z level=INFO msg="Uploading 1 narinfos"5068lb # [ 91.465282] niks3-server[1035]: 2026/09/22 10:37:28 INFO Signed narinfos id=25 count=15069lb # [ 91.475067] niks3-server[1035]: 2026/09/22 10:37:28 INFO Registered completed upload object_key=n7pcsnwv1a8b1wwcq715iprr5f6c4mcr.ls5070worker2 # [ 91.499442] nix-grpc-daemon[879]: time=2026-09-22T10:37:28.143Z level=INFO msg="Upload complete. (502ms)"5071lb # [ 91.482111] niks3-server[1035]: 2026/09/22 10:37:28 INFO Received complete upload request method=POST path=/api/pending_closures/25/complete5072lb # [ 91.487330] niks3-server[1035]: 2026/09/22 10:37:28 INFO Completed upload id=255073lb # [ 91.490526] niks3-server[1035]: 2026/09/22 10:37:28 INFO Registered completed upload object_key=n7pcsnwv1a8b1wwcq715iprr5f6c4mcr.narinfo5074worker2 # [ 91.508292] nix-grpc-daemon[777]: ts=2026-09-22T10:37:28Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_in=5685075worker2 # [ 91.519862] nix-grpc-daemon[777]: ts=2026-09-22T10:37:28Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:411445076worker2 # [ 91.526941] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)5077worker2 # [ 91.531520] nix-daemon[864]: reaped child process 1205, status = succeeded5078client # building '/nix/store/n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv' on lb:50051...5079worker2 # [ 91.544194] nix-daemon[864]: accepted connection from pid 777, user nix-grpc-daemon (trusted)5080client # uses-input-w1onlyc> worker2: building n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv5081client # uses-input-w1onlyc> disabling binary cache 'https://cache.nixos.org' for 60 seconds5082worker2 # [ 92.986547] nix-daemon[864]: reaped child process 1219, status = succeeded5083worker2 # [ 93.167946] nix-daemon[864]: accepted connection from pid 1235, user nix-grpc-daemon (trusted)5084worker2 # [ 93.187303] nix-daemon[864]: reaped child process 1238, status = succeeded5085lb # [ 93.182051] niks3-server[1035]: 2026/09/22 10:37:29 INFO Received uploads request method=POST path=/api/pending_closures5086worker2 # [ 93.211676] nix-grpc-daemon[879]: time=2026-09-22T10:37:29.854Z level=INFO msg="Uploading 0 paths to lb (1 already cached)"5087lb # [ 93.218165] niks3-server[1035]: 2026/09/22 10:37:29 INFO Registered completed upload object_key=13r5ipj38sk1irdi724wm1mbl9lw720b.ls5088worker2 # [ 93.246772] nix-grpc-daemon[879]: time=2026-09-22T10:37:29.890Z level=INFO msg="Uploading 1 narinfos"5089lb # [ 93.229455] niks3-server[1035]: 2026/09/22 10:37:29 INFO Received sign narinfos request method=POST path=/api/pending_closures/26/sign5090lb # [ 93.239717] niks3-server[1035]: 2026/09/22 10:37:29 INFO Signed narinfos id=26 count=15091lb # [ 93.245691] niks3-server[1035]: 2026/09/22 10:37:29 INFO Registered completed upload object_key=log/n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv5092lb # [ 93.251812] niks3-server[1035]: 2026/09/22 10:37:29 INFO Received complete upload request method=POST path=/api/pending_closures/26/complete5093worker2 # [ 93.275430] nix-grpc-daemon[879]: time=2026-09-22T10:37:29.919Z level=INFO msg="Upload complete. (320ms)"5094lb # [ 93.258217] niks3-server[1035]: 2026/09/22 10:37:29 INFO Completed upload id=265095lb # [ 93.260984] niks3-server[1035]: 2026/09/22 10:37:29 INFO Registered completed upload object_key=13r5ipj38sk1irdi724wm1mbl9lw720b.narinfo5096worker2 # [ 93.284867] nix-grpc-daemon[777]: ts=2026-09-22T10:37:29Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=1 drv=n7pcsnwv1a8b1wwcq715iprr5f6c4mcr-uses-input-w1onlyc.drv assign_id=12 outputs=15097worker2 # [ 93.302469] nix-daemon[864]: reaped child process 1217, status = succeeded5098client # copying 1 paths...5099worker2 # [ 93.335068] nix-grpc-daemon[777]: ts=2026-09-22T10:37:29Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:411445100client # copying path '/nix/store/13r5ipj38sk1irdi724wm1mbl9lw720b-uses-input-w1onlyc' from 'grpc://lb:50051'...5101worker2 # [ 93.368124] nix-grpc-daemon[777]: ts=2026-09-22T10:37:30Z level=debug event=rpc_start method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:411445102worker2 # [ 93.374609] nix-grpc-daemon[777]: ts=2026-09-22T10:37:30Z level=info event=rpc method=FetchNars cn=ci-1 peer=ipv4:192.168.1.2:41144 duration_s=0 paths=1 nar_bytes_out=1205103client # [ 93.463135] nix-daemon[1040]: reaped child process 1170, status = succeeded5104client: (finished: must succeed: NIX_REMOTE=daemon nix build -L --max-jobs 0 --builders 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&system=aarch64-linux aarch64-linux - 4' --no-link -f /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix job --argstr tag w1only --argstr salt c >&2, in 3.09 seconds)5105(finished: subtest: an input only one worker has still reaches the builder, in 10.72 seconds)5106subtest: upload whose reference the worker lacks is completed from the cache5107client: must succeed: nix-build --no-out-link /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix -A input --argstr tag viacache5108client # this derivation will be built:5109client # /nix/store/x1rrnjihpa1xvpyxxxdrjzp3c8z8nayq-local-input-viacache.drv5110client # building '/nix/store/x1rrnjihpa1xvpyxxxdrjzp3c8z8nayq-local-input-viacache.drv'...5111client: (finished: must succeed: nix-build --no-out-link /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix -A input --argstr tag viacache, in 0.45 seconds)5112client: must succeed: nix-build --no-out-link /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix -A referrer --argstr tag viacache5113client # this derivation will be built:5114client # /nix/store/1m587dk4visgyryf6n0mgp2rlp7pbjd1-refers-to-input-viacache.drv5115client # building '/nix/store/1m587dk4visgyryf6n0mgp2rlp7pbjd1-refers-to-input-viacache.drv'...5116client: (finished: must succeed: nix-build --no-out-link /nix/store/d2v6gfscfnwmpkifqgfkvs9vxxfraw1i-dep.nix -A referrer --argstr tag viacache, in 0.43 seconds)5117client: must succeed: nix-store --export /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache > /tmp/shared/ref.closure5118client: (finished: must succeed: nix-store --export /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache > /tmp/shared/ref.closure, in 0.19 seconds)5119worker1: must succeed: nix-store --import < /tmp/shared/ref.closure5120worker1: (finished: must succeed: nix-store --import < /tmp/shared/ref.closure, in 0.23 seconds)5121worker2 # [ 94.887668] systemd[1]: Stopping Nix worker-protocol over gRPC...5122worker2 # [ 95.298763] nix-grpc-daemon[777]: ts=2026-09-22T10:37:31Z level=info event=signal_exit5123lb # [ 95.522262] niks3-server[1035]: 2026/09/22 10:37:32 INFO lead: released remote=192.168.1.4:407505124worker1 # [ 96.503389] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_restarting addr=lb:500515125worker2 # [ 96.543521] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_restarting5126worker2 # [ 96.543964] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:411285127worker2 # [ 96.554472] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=worker_gone addr=192.168.1.3:500515128worker2 # [ 96.559006] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_restarting addr=lb:500515129worker2 # [ 96.563520] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:411285130worker2 # [ 96.568517] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=worker_gone addr=192.168.1.4:500515131worker2 # [ 96.594895] nix-daemon[864]: reaped child process 866, status = succeeded5132worker1 # [ 96.703419] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_connected addr=lb:500515133worker1 # [ 96.708911] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5134worker2 # [ 96.765194] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=niks3_push_exited status=succeeded why="niks3 push exited"5135worker1 # [ 96.909754] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_connected addr=lb:500515136worker1 # [ 96.914948] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5137worker1 # [ 97.115615] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_connected addr=lb:500515138worker1 # [ 97.120823] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5139worker1 # [ 97.161893] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_take_over5140lb # [ 97.179572] niks3-server[1035]: 2026/09/22 10:37:33 INFO lead: acquired remote=192.168.1.3:582925141worker1 # [ 97.321476] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=scheduler_connected addr=lb:500515142worker1 # [ 97.341322] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=worker_session_open cn=worker-1 peer=ipv4:192.168.1.2:363365143worker1 # [ 97.346525] nix-grpc-daemon[777]: ts=2026-09-22T10:37:33Z level=info event=worker_hello addr=192.168.1.3:50051 systems=aarch64-linux max_jobs=1 running=05144worker2 # [ 97.398631] nix-grpc-daemon[777]: E0000 00:00:1790073454.042664 822 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 18, started true, shutdown true5145worker2 # [ 97.398968] nix-grpc-daemon[777]: E0000 00:00:1790073454.042947 822 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 7, started true, shutdown true5146worker2 # [ 98.234810] systemd[1]: nix-grpc-daemon.service: Deactivated successfully.5147worker2 # [ 98.241649] systemd[1]: Stopped Nix worker-protocol over gRPC.5148worker2 # [ 98.246774] systemd[1]: nix-grpc-daemon.service: Consumed 3.382s CPU time over 1min 18.273s wall clock time, 63.6M memory peak, 99.4K incoming IP traffic, 274.3K outgoing IP traffic.5149worker2 # [ 98.258240] systemd[1]: nix-grpc-daemon.socket: Deactivated successfully.5150worker2 # [ 98.263243] systemd[1]: Closed Nix worker-protocol over gRPC.5151worker2 # [ 98.266592] systemd[1]: nix-grpc-daemon.socket: Consumed 599us CPU time over 1min 26.647s wall clock time, 256K memory peak, 107.2K incoming IP traffic, 88.4K outgoing IP traffic.5152lb: waiting until exactly ['worker2'] endpoints of aarch64-linux are unhealthy5153lb: must succeed: curl -sf localhost:9901/clusters5154lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5155lb: (finished: waiting until exactly ['worker2'] endpoints of aarch64-linux are unhealthy, in 0.08 seconds)5156client: must succeed: nix path-info --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache >&25157worker1 # [ 98.590279] nix-grpc-daemon[777]: ts=2026-09-22T10:37:35Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:330685158worker1 # [ 98.596818] nix-grpc-daemon[777]: ts=2026-09-22T10:37:35Z level=debug event=rpc_start method=QueryPathInfos cn=ci-1 peer=ipv4:192.168.1.2:330685159client # /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache5160client: (finished: must succeed: nix path-info --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache >&2, in 0.34 seconds)5161worker2: must fail: test -e /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache5162worker2: (finished: must fail: test -e /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache, in 0.04 seconds)5163worker2 # [ 98.806579] systemd[1]: Listening on Nix worker-protocol over gRPC.5164worker1 # [ 98.851158] systemd[1]: Stopping Nix worker-protocol over gRPC...5165worker1 # [ 99.155164] nix-grpc-daemon[777]: ts=2026-09-22T10:37:35Z level=info event=signal_exit5166worker2 # [ 99.221618] systemd[1]: Starting scheduler-token.service...5167worker2 # [ 99.336667] systemd[1]: scheduler-token.service: Deactivated successfully.5168worker2 # [ 99.344863] systemd[1]: Finished scheduler-token.service.5169worker2 # [ 99.346719] systemd[1]: scheduler-token.service: Consumed 54ms CPU time over 118ms wall clock time, 2.1M memory peak, 925B incoming IP traffic, 452B outgoing IP traffic.5170worker2 # [ 99.354828] systemd[1]: Starting Nix worker-protocol over gRPC...5171lb # [ 99.425464] niks3-server[1035]: 2026/09/22 10:37:36 INFO lead: released remote=192.168.1.3:582925172worker2 # [ 99.544407] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=warning msg="--niks3 http:// sends the bearer token in clear"5173worker2 # [ 99.547032] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=oidc_provider name=mock issuer=http://192.168.1.2:8080/oidc5174worker2 # [ 99.551208] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=roles builder=1 scheduler=1 scheduler_addr=lb:50051 advertise=192.168.1.4:50051 niks3=http://lb:5751 max_jobs=15175worker2 # [ 99.560072] nix-grpc-daemon[1262]: WARNING: All log messages before absl::InitializeLog() is called are written to STDERR5176worker2 # [ 99.561547] nix-grpc-daemon[1262]: E0000 00:00:1790073456.202396 1262 tls_security_connector.cc:771] TlsServerCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5177worker2 # [ 99.573571] nix-grpc-daemon[1262]: E0000 00:00:1790073456.208746 1262 tls_security_connector.cc:767] TlsServerCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5178worker2 # [ 99.578477] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=startup listen=systemd proxy_socket=/nix/var/nix/daemon-socket/socket5179worker2 # [ 99.581029] systemd[1]: Started Nix worker-protocol over gRPC.5180lb # [ 99.561043] niks3-server[1035]: 2026/09/22 10:37:36 INFO lead: acquired remote=192.168.1.4:344265181worker2 # [ 99.582796] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_take_over5182worker2 # [ 99.586589] nix-grpc-daemon[1262]: E0000 00:00:1790073456.230429 1301 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5183worker2 # [ 99.616126] nix-grpc-daemon[1262]: E0000 00:00:1790073456.250208 1301 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5184worker2 # [ 99.618470] nix-grpc-daemon[1262]: E0000 00:00:1790073456.250369 1301 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5185worker2 # [ 99.626397] nix-grpc-daemon[1262]: E0000 00:00:1790073456.270001 1301 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5186worker2 # [ 99.638645] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_connected addr=lb:500515187worker2 # [ 99.640362] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5188lb # [ 99.808461] envoy[896]: conn peer=[2001:db8:1::1]:32860 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=3739 tx=2122 ms=33265189lb # [ 99.813290] envoy[896]: conn peer=[2001:db8:1::1]:32870 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=448 tx=328 ms=665190lb # [ 99.817887] envoy[896]: conn peer=[2001:db8:1::1]:32882 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=3770 tx=2139 ms=25925191lb # [ 99.822671] envoy[896]: conn peer=[2001:db8:1::1]:48160 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=448 tx=328 ms=695192worker2 # [ 99.841725] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_connected addr=lb:500515193lb # [ 99.828891] envoy[896]: conn peer=[2001:db8:1::4]:49646 tls=TLSv1.3 subject="CN=worker-2" sni=- flags=- tls_fail="-" rx=83941 tx=4345 ms=764215194worker2 # [ 99.847327] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5195lb # [ 99.834602] envoy[896]: conn peer=[2001:db8:1::1]:48168 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=591 tx=523 ms=285196worker2 # [ 100.048553] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_connected addr=lb:500515197worker2 # [ 100.053489] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5198worker2 # [ 100.254681] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_connected addr=lb:500515199worker2 # [ 100.259873] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:36Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5200worker1 # [ 100.408554] nix-grpc-daemon[777]: ts=2026-09-22T10:37:37Z level=info event=scheduler_restarting5201worker1 # [ 100.409124] nix-grpc-daemon[777]: ts=2026-09-22T10:37:37Z level=info event=scheduler_restarting addr=lb:500515202worker1 # [ 100.409306] nix-grpc-daemon[777]: ts=2026-09-22T10:37:37Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:363365203worker1 # [ 100.409424] nix-grpc-daemon[777]: ts=2026-09-22T10:37:37Z level=info event=worker_gone addr=192.168.1.3:500515204worker2 # [ 100.460988] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:37Z level=info event=scheduler_connected addr=lb:500515205worker2 # [ 100.466080] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:37Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5206worker1 # [ 100.447267] nix-daemon[844]: reaped child process 846, status = succeeded5207worker1 # [ 100.625440] nix-grpc-daemon[777]: ts=2026-09-22T10:37:37Z level=info event=niks3_push_exited status=succeeded why="niks3 push exited"5208worker2 # [ 100.668467] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:37Z level=info event=scheduler_connected addr=lb:500515209worker2 # [ 100.684125] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:37Z level=info event=oidc_jwks provider=mock keys=15210worker2 # [ 100.689401] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:37Z level=info event=worker_session_open cn=oidc:mock:node:worker2 peer=ipv4:192.168.1.2:372125211worker2 # [ 100.694703] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:37Z level=info event=worker_hello addr=192.168.1.4:50051 systems=aarch64-linux max_jobs=1 running=05212worker1 # [ 101.300664] nix-grpc-daemon[777]: E0000 00:00:1790073457.931296 822 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 15, started true, shutdown true5213worker1 # [ 101.301062] nix-grpc-daemon[777]: E0000 00:00:1790073457.931591 822 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 7, started true, shutdown true5214worker1 # [ 102.121617] systemd[1]: nix-grpc-daemon.service: Deactivated successfully.5215worker1 # [ 102.127832] systemd[1]: Stopped Nix worker-protocol over gRPC.5216worker1 # [ 102.135036] systemd[1]: nix-grpc-daemon.service: Consumed 2.737s CPU time over 1min 22.329s wall clock time, 55.7M memory peak, 73.1K incoming IP traffic, 136.4K outgoing IP traffic.5217worker1 # [ 102.152997] systemd[1]: nix-grpc-daemon.socket: Deactivated successfully.5218worker1 # [ 102.155060] systemd[1]: Closed Nix worker-protocol over gRPC.5219worker1 # [ 102.156919] systemd[1]: nix-grpc-daemon.socket: Consumed 614us CPU time over 1min 31.182s wall clock time, 256K memory peak, 86.8K incoming IP traffic, 79.6K outgoing IP traffic.5220lb: waiting until exactly ['worker1'] endpoints of aarch64-linux are unhealthy5221lb: must succeed: curl -sf localhost:9901/clusters5222lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5223lb: (finished: waiting until exactly ['worker1'] endpoints of aarch64-linux are unhealthy, in 0.08 seconds)5224client: must succeed: nix copy --no-check-sigs --to 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' /nix/store/4rig83zwv4xj1zajs08ima13w0yh676v-refers-to-input-viacache >&25225worker2 # [ 102.543142] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:39Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:607465226worker2 # [ 102.558694] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5227worker2 # [ 102.584859] nix-grpc-daemon[1262]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 12 ms (attempt 1/5)5228worker2 # [ 102.603712] nix-grpc-daemon[1262]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 127 ms (attempt 2/5)5229worker2 # [ 102.737919] nix-grpc-daemon[1262]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 197 ms (attempt 3/5)5230worker2 # [ 102.942494] nix-grpc-daemon[1262]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 347 ms (attempt 4/5)5231worker2 # [ 103.297521] nix-grpc-daemon[1262]: disabling binary cache 'https://cache.nixos.org' for 60 seconds5232worker2 # [ 103.303218] nix-grpc-daemon[1262]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org5233client # copying 2 paths...5234client # copying path '/nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache' to 'grpc://lb:50051'...5235worker2 # [ 103.327008] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:39Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:607465236client # copying path '/nix/store/4rig83zwv4xj1zajs08ima13w0yh676v-refers-to-input-viacache' to 'grpc://lb:50051'...5237worker2 # [ 103.344754] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5238worker2 # [ 103.372870] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:40Z level=info event=niks3_push_started pid=13225239worker2 # [ 103.582542] nix-grpc-daemon[1322]: time=2026-09-22T10:37:40.226Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.key ca=""5240worker2 # [ 103.758807] nix-daemon[864]: accepted connection from pid 1335, user nix-grpc-daemon (trusted)5241worker2 # [ 103.775879] nix-daemon[864]: reaped child process 1338, status = succeeded5242lb # [ 103.773981] niks3-server[1035]: 2026/09/22 10:37:40 INFO Received uploads request method=POST path=/api/pending_closures5243lb # [ 103.785301] niks3-server[1035]: 2026/09/22 10:37:40 INFO Received uploads request method=POST path=/api/pending_closures5244worker2 # [ 103.813815] nix-grpc-daemon[1322]: time=2026-09-22T10:37:40.456Z level=INFO msg="Uploading 2 paths to lb (0 already cached)"5245worker2 # [ 103.816479] nix-grpc-daemon[1322]: time=2026-09-22T10:37:40.456Z level=INFO msg="Uploading syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache (128B)"5246worker2 # [ 103.818258] nix-grpc-daemon[1322]: time=2026-09-22T10:37:40.456Z level=INFO msg="Uploading 4rig83zwv4xj1zajs08ima13w0yh676v-refers-to-input-viacache (184B)"5247lb # [ 103.828708] niks3-server[1035]: 2026/09/22 10:37:40 INFO Registered completed upload object_key=nar/1rf1rg73pvnaa9kk6sy9r1ydj24b4g0kb0wnx0z5hxl1nvkmlavq.nar.zst5248lb # [ 103.848101] niks3-server[1035]: 2026/09/22 10:37:40 INFO Registered completed upload object_key=4rig83zwv4xj1zajs08ima13w0yh676v.ls5249lb # [ 103.858027] niks3-server[1035]: 2026/09/22 10:37:40 INFO Registered completed upload object_key=nar/1kmnh5f2l7jjcgaq9b9988bjix6z8zjf6v350xllq5hlzgnb1f75.nar.zst5250lb # [ 103.863945] niks3-server[1035]: 2026/09/22 10:37:40 INFO Received sign narinfos request method=POST path=/api/pending_closures/28/sign5251worker2 # [ 103.888224] nix-grpc-daemon[1322]: time=2026-09-22T10:37:40.531Z level=INFO msg="Uploading 3 narinfos"5252lb # [ 103.871253] niks3-server[1035]: 2026/09/22 10:37:40 INFO Signed narinfos id=28 count=25253lb # [ 103.874169] niks3-server[1035]: 2026/09/22 10:37:40 INFO Received sign narinfos request method=POST path=/api/pending_closures/27/sign5254lb # [ 103.879120] niks3-server[1035]: 2026/09/22 10:37:40 INFO Signed narinfos id=27 count=15255lb # [ 103.883517] niks3-server[1035]: 2026/09/22 10:37:40 INFO Registered completed upload object_key=syvivri0wisb7fd33rxcg9kz63kc2bv7.ls5256lb # [ 103.905488] niks3-server[1035]: 2026/09/22 10:37:40 INFO Registered completed upload object_key=syvivri0wisb7fd33rxcg9kz63kc2bv7.narinfo5257lb # [ 103.915290] niks3-server[1035]: 2026/09/22 10:37:40 INFO Received complete upload request method=POST path=/api/pending_closures/27/complete5258worker2 # [ 103.942887] nix-grpc-daemon[1322]: time=2026-09-22T10:37:40.587Z level=INFO msg="Upload complete. (355ms)"5259lb # [ 103.926835] niks3-server[1035]: 2026/09/22 10:37:40 INFO Completed upload id=275260lb # [ 103.933535] niks3-server[1035]: 2026/09/22 10:37:40 INFO Received complete upload request method=POST path=/api/pending_closures/28/complete5261lb # [ 103.941128] niks3-server[1035]: 2026/09/22 10:37:40 INFO Completed upload id=285262worker2 # [ 103.957202] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:40Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:60746 duration_s=0 paths=2 nar_bytes_in=3125263lb # [ 103.946313] niks3-server[1035]: 2026/09/22 10:37:40 INFO Registered completed upload object_key=4rig83zwv4xj1zajs08ima13w0yh676v.narinfo5264worker2 # [ 103.973632] nix-daemon[864]: reaped child process 1320, status = succeeded5265client: (finished: must succeed: nix copy --no-check-sigs --to 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' /nix/store/4rig83zwv4xj1zajs08ima13w0yh676v-refers-to-input-viacache >&2, in 1.71 seconds)5266worker2: must succeed: test -e /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache && test -e /nix/store/4rig83zwv4xj1zajs08ima13w0yh676v-refers-to-input-viacache5267worker2: (finished: must succeed: test -e /nix/store/syvivri0wisb7fd33rxcg9kz63kc2bv7-local-input-viacache && test -e /nix/store/4rig83zwv4xj1zajs08ima13w0yh676v-refers-to-input-viacache, in 0.03 seconds)5268worker1 # [ 104.072989] systemd[1]: Listening on Nix worker-protocol over gRPC.5269worker1 # [ 104.078792] systemd[1]: Starting Nix worker-protocol over gRPC...5270worker1 # [ 104.276487] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:40Z level=info event=warning msg="--niks3 http:// sends the bearer token in clear"5271worker1 # [ 104.279397] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:40Z level=info event=oidc_provider name=mock issuer=http://192.168.1.2:8080/oidc5272worker1 # [ 104.283768] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:40Z level=info event=roles builder=1 scheduler=1 scheduler_addr=lb:50051 advertise=192.168.1.3:50051 niks3=http://lb:5751 max_jobs=15273worker1 # [ 104.291538] nix-grpc-daemon[1141]: WARNING: All log messages before absl::InitializeLog() is called are written to STDERR5274worker1 # [ 104.306875] nix-grpc-daemon[1141]: E0000 00:00:1790073460.921658 1141 tls_security_connector.cc:771] TlsServerCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5275worker1 # [ 104.311063] nix-grpc-daemon[1141]: E0000 00:00:1790073460.925267 1141 tls_security_connector.cc:767] TlsServerCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5276lb: waiting until exactly no endpoints of aarch64-linux are unhealthy5277lb: must succeed: curl -sf localhost:9901/clusters5278worker1 # [ 104.320189] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:40Z level=info event=startup listen=systemd proxy_socket=/nix/var/nix/daemon-socket/socket5279worker1 # [ 104.348200] systemd[1]: Started Nix worker-protocol over gRPC.5280worker1 # [ 104.349047] nix-grpc-daemon[1141]: E0000 00:00:1790073460.949062 1182 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5281worker1 # [ 104.354691] nix-grpc-daemon[1141]: E0000 00:00:1790073460.966932 1182 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5282worker1 # [ 104.357529] nix-grpc-daemon[1141]: E0000 00:00:1790073460.967114 1182 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5283worker1 # [ 104.360111] nix-grpc-daemon[1141]: E0000 00:00:1790073460.985002 1182 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5284worker1 # [ 104.367438] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:40Z level=info event=scheduler_connected addr=lb:500515285worker2 # [ 104.404757] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:41Z level=info event=worker_session_open cn=worker-1 peer=ipv4:192.168.1.2:372125286worker2 # [ 104.410815] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:41Z level=info event=worker_hello addr=192.168.1.3:50051 systems=aarch64-linux max_jobs=1 running=05287lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.09 seconds)5288lb: (finished: waiting until exactly no endpoints of aarch64-linux are unhealthy, in 0.09 seconds)5289(finished: subtest: upload whose reference the worker lacks is completed from the cache, in 10.92 seconds)5290subtest: a niks3 restart does not cost the leader its role5291lb: must succeed: curl -sf localhost:9901/clusters5292lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.07 seconds)5293lb: must succeed: curl -sf localhost:9901/clusters5294lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)5295lb # [ 104.581390] niks3-server[1035]: 2026/09/22 10:37:41 INFO Shutdown signal received, draining in-flight requests timeout=10s5296worker1 # [ 104.567503] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:41Z level=info event=scheduler_lock_lost status=2005297lb # [ 104.588433] systemd[1]: Stopping niks3 server...5298lb # [ 104.590097] niks3-server[1035]: 2026/09/22 10:37:41 INFO lead: released remote=192.168.1.4:344265299worker2 # [ 104.605319] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:41Z level=info event=scheduler_lock_lost status=2005300lb # [ 109.335917] systemd[1]: niks3.service: Deactivated successfully.5301lb # [ 109.342799] systemd[1]: Stopped niks3 server.5302lb # [ 109.347383] systemd[1]: niks3.service: Consumed 562ms CPU time over 1min 17.514s wall clock time, 22.5M memory peak, 79.5K incoming IP traffic, 112.4K outgoing IP traffic.5303lb # [ 109.371919] systemd[1]: Starting niks3 server...5304lb # [ 109.484390] niks3-server[1173]: 2026/09/22 10:37:46 goose: no migrations to run. current version: 202609200000005305lb # [ 109.491691] niks3-server[1173]: 2026/09/22 10:37:46 OK 1_commit_pending_closure.sql (8.21ms)5306lb # [ 109.495850] niks3-server[1173]: 2026/09/22 10:37:46 OK 2_object_stats_trigger.sql (3.79ms)5307lb # [ 109.497476] niks3-server[1173]: 2026/09/22 10:37:46 goose: up to current file version: 25308lb # [ 109.499650] niks3-server[1173]: 2026/09/22 10:37:46 INFO Loaded signing key name=farm-test-1 path=/nix/store/8lzz3wcf17v05vfdp1zcfaa9hipw3f70-key5309lb # [ 109.511536] niks3-server[1173]: 2026/09/22 10:37:46 INFO Read proxy enabled — serving cache objects from S35310lb # [ 109.516436] niks3-server[1173]: 2026/09/22 10:37:46 INFO Using socket-activated listener address=0.0.0.0:57515311lb # [ 109.517711] niks3-server[1173]: 2026/09/22 10:37:46 INFO systemd watchdog enabled interval=15s5312lb # [ 109.518765] niks3-server[1173]: 2026/09/22 10:37:46 INFO Starting HTTP server address=0.0.0.0:57515313lb # [ 109.519882] niks3-server[1173]: 2026/09/22 10:37:46 INFO lead: acquired remote=192.168.1.4:431145314lb # [ 109.522866] systemd[1]: Started niks3 server.5315lb: waiting for unit niks3.service5316lb: (finished: waiting for unit niks3.service, in 0.04 seconds)5317lb: must succeed: curl -sf localhost:9901/clusters5318lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)5319worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics5320worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)5321lb: must succeed: curl -sf localhost:9901/clusters5322lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.07 seconds)5323lb: must succeed: curl -sf localhost:9901/clusters5324lb # [ 109.810026] envoy[896]: conn peer=[2001:db8:1::3]:51654 tls=TLSv1.3 subject="CN=worker-1" sni=- flags=- tls_fail="-" rx=16654 tx=4374 ms=804235325lb # [ 109.812984] envoy[896]: conn peer=[2001:db8:1::1]:49060 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=1006 tx=305 ms=14295326lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)5327worker2: must fail: journalctl -u nix-grpc-daemon | grep -q event=scheduler_yield5328worker2: (finished: must fail: journalctl -u nix-grpc-daemon | grep -q event=scheduler_yield, in 0.07 seconds)5329client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag after-niks3-restart >&25330worker2 # [ 110.192677] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:46Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:607465331client # don't know how to build these paths:5332client # /nix/store/6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv5333worker1 # [ 110.174988] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:46Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:457385334worker1 # [ 110.190735] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5335worker1 # [ 110.220355] nix-grpc-daemon[1141]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 31 ms (attempt 1/5)5336worker1 # [ 110.258429] nix-grpc-daemon[1141]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 42 ms (attempt 2/5)5337worker1 # [ 110.307957] nix-grpc-daemon[1141]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 358 ms (attempt 3/5)5338worker1 # [ 110.673667] nix-grpc-daemon[1141]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 637 ms (attempt 4/5)5339worker1 # [ 111.322230] nix-grpc-daemon[1141]: disabling binary cache 'https://cache.nixos.org' for 60 seconds5340worker1 # [ 111.333081] nix-grpc-daemon[1141]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org5341worker2 # [ 111.488154] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:48Z level=debug event=want client=1 drv=mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv cached=05342worker2 # [ 111.489125] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:48Z level=debug event=assigned drv=mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv worker=192.168.1.4:50051 clients=15343worker2 # [ 111.534244] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:48Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:607465344worker2 # [ 111.575095] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:48Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:607465345client # copying 1 paths to 'lb:50051'...5346client # copying path '/nix/store/mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv' to 'lb:50051'...5347worker2 # [ 111.599069] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:48Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:607465348worker2 # [ 111.621540] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5349worker2 # [ 111.854024] nix-daemon[864]: accepted connection from pid 1357, user nix-grpc-daemon (trusted)5350worker2 # [ 111.871932] nix-daemon[864]: reaped child process 1360, status = succeeded5351worker2 # [ 112.042098] nix-daemon[864]: accepted connection from pid 1362, user nix-grpc-daemon (trusted)5352worker2 # [ 112.058766] nix-daemon[864]: reaped child process 1365, status = succeeded5353lb # [ 112.058139] niks3-server[1173]: 2026/09/22 10:37:48 INFO Received uploads request method=POST path=/api/pending_closures5354worker2 # [ 112.093464] nix-grpc-daemon[1322]: time=2026-09-22T10:37:48.736Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"5355worker2 # [ 112.094943] nix-grpc-daemon[1322]: time=2026-09-22T10:37:48.736Z level=INFO msg="Uploading mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv (544B)"5356lb # [ 112.111545] niks3-server[1173]: 2026/09/22 10:37:48 INFO Received sign narinfos request method=POST path=/api/pending_closures/29/sign5357lb # [ 112.124382] niks3-server[1173]: 2026/09/22 10:37:48 INFO Registered completed upload object_key=nar/0hn539w6achi7azzyhpihsi61g85x9f535dxv438yq5y50ksrp1d.nar.zst5358lb # [ 112.151895] niks3-server[1173]: 2026/09/22 10:37:48 INFO Signed narinfos id=29 count=15359worker2 # [ 112.173169] nix-grpc-daemon[1322]: time=2026-09-22T10:37:48.816Z level=INFO msg="Uploading 1 narinfos"5360lb # [ 112.166845] niks3-server[1173]: 2026/09/22 10:37:48 INFO Registered completed upload object_key=mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z.ls5361lb # [ 112.173562] niks3-server[1173]: 2026/09/22 10:37:48 INFO Received complete upload request method=POST path=/api/pending_closures/29/complete5362lb # [ 112.180254] niks3-server[1173]: 2026/09/22 10:37:48 INFO Registered completed upload object_key=mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z.narinfo5363lb # [ 112.185604] niks3-server[1173]: 2026/09/22 10:37:48 INFO Completed upload id=295364worker2 # [ 112.203973] nix-grpc-daemon[1322]: time=2026-09-22T10:37:48.848Z level=INFO msg="Upload complete. (552ms)"5365worker2 # [ 112.214358] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:48Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:60746 duration_s=0 paths=1 nar_bytes_in=5445366worker2 # [ 112.232432] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:48Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:607465367worker2 # [ 112.244927] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5368worker2 # [ 112.248482] nix-daemon[864]: reaped child process 1355, status = succeeded5369client # building '/nix/store/mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv' on lb:50051...5370client # farm-a> worker2: building mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv5371worker2 # [ 112.265204] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5372client # farm-a> disabling binary cache 'https://cache.nixos.org' for 60 seconds5373client # farm-a> LOG-farm-a-after-niks3-restart5374worker2 # [ 113.287107] nix-daemon[864]: reaped child process 1369, status = succeeded5375worker2 # [ 113.447355] nix-daemon[864]: accepted connection from pid 1385, user nix-grpc-daemon (trusted)5376worker2 # [ 113.464856] nix-daemon[864]: reaped child process 1388, status = succeeded5377lb # [ 113.461263] niks3-server[1173]: 2026/09/22 10:37:50 INFO Received uploads request method=POST path=/api/pending_closures5378worker2 # [ 113.492382] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.135Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"5379worker2 # [ 113.493873] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.135Z level=INFO msg="Uploading j3iswx3ynilvcycpyw0z667966c9kpzx-farm-a (144B)"5380lb # [ 113.499492] niks3-server[1173]: 2026/09/22 10:37:50 INFO Registered completed upload object_key=log/mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv5381lb # [ 113.513036] niks3-server[1173]: 2026/09/22 10:37:50 INFO Registered completed upload object_key=nar/1diw08qzlsh0zkrx4kvj1mvbc57g7r65j1f459ankpg9yc5rgcpx.nar.zst5382lb # [ 113.525458] niks3-server[1173]: 2026/09/22 10:37:50 INFO Received sign narinfos request method=POST path=/api/pending_closures/30/sign5383worker2 # [ 113.549457] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.192Z level=INFO msg="Uploading 1 narinfos"5384lb # [ 113.534680] niks3-server[1173]: 2026/09/22 10:37:50 INFO Signed narinfos id=30 count=15385lb # [ 113.540156] niks3-server[1173]: 2026/09/22 10:37:50 INFO Registered completed upload object_key=j3iswx3ynilvcycpyw0z667966c9kpzx.ls5386lb # [ 113.547816] niks3-server[1173]: 2026/09/22 10:37:50 INFO Received complete upload request method=POST path=/api/pending_closures/30/complete5387lb # [ 113.556781] niks3-server[1173]: 2026/09/22 10:37:50 INFO Registered completed upload object_key=j3iswx3ynilvcycpyw0z667966c9kpzx.narinfo5388lb # [ 113.564075] niks3-server[1173]: 2026/09/22 10:37:50 INFO Completed upload id=305389worker2 # [ 113.582334] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.226Z level=INFO msg="Upload complete. (305ms)"5390worker2 # [ 113.601622] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:60746 duration_s=1 drv=mxnzwd2qgmj6r8zhjk6jlcxv8j46h04z-farm-a.drv assign_id=1 outputs=15391worker2 # [ 113.621270] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=debug event=want client=1 drv=5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv cached=05392worker2 # [ 113.625721] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=debug event=assigned drv=5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv worker=192.168.1.4:50051 clients=15393worker2 # [ 113.630211] nix-daemon[864]: reaped child process 1367, status = succeeded5394worker2 # [ 113.658185] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:607465395worker2 # [ 113.673336] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:607465396worker2 # [ 113.687807] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.332Z level=INFO msg="All 1 paths already cached"5397client # copying 1 paths to 'lb:50051'...5398client # copying path '/nix/store/5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv' to 'lb:50051'...5399worker2 # [ 113.698337] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:607465400worker2 # [ 113.712190] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5401worker2 # [ 113.918138] nix-daemon[864]: accepted connection from pid 1394, user nix-grpc-daemon (trusted)5402worker2 # [ 113.936328] nix-daemon[864]: reaped child process 1397, status = succeeded5403worker2 # [ 114.108534] nix-daemon[864]: accepted connection from pid 1399, user nix-grpc-daemon (trusted)5404worker2 # [ 114.126534] nix-daemon[864]: reaped child process 1402, status = succeeded5405lb # [ 114.119775] niks3-server[1173]: 2026/09/22 10:37:50 INFO Received uploads request method=POST path=/api/pending_closures5406worker2 # [ 114.150960] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.794Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"5407worker2 # [ 114.152532] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.795Z level=INFO msg="Uploading 5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv (656B)"5408lb # [ 114.151930] niks3-server[1173]: 2026/09/22 10:37:50 INFO Registered completed upload object_key=nar/1xpcisa10g5by3yknnrs1xwhxhf4s243h7gwzjii0799pwbhfjfa.nar.zst5409lb # [ 114.167403] niks3-server[1173]: 2026/09/22 10:37:50 INFO Received sign narinfos request method=POST path=/api/pending_closures/31/sign5410worker2 # [ 114.192348] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.836Z level=INFO msg="Uploading 1 narinfos"5411lb # [ 114.180407] niks3-server[1173]: 2026/09/22 10:37:50 INFO Signed narinfos id=31 count=15412lb # [ 114.183673] niks3-server[1173]: 2026/09/22 10:37:50 INFO Registered completed upload object_key=5ml8wmsr73y0fh0dflaimhsh2xbcd3ch.ls5413lb # [ 114.194415] niks3-server[1173]: 2026/09/22 10:37:50 INFO Received complete upload request method=POST path=/api/pending_closures/31/complete5414lb # [ 114.201983] niks3-server[1173]: 2026/09/22 10:37:50 INFO Registered completed upload object_key=5ml8wmsr73y0fh0dflaimhsh2xbcd3ch.narinfo5415lb # [ 114.206515] niks3-server[1173]: 2026/09/22 10:37:50 INFO Completed upload id=315416worker2 # [ 114.224382] nix-grpc-daemon[1322]: time=2026-09-22T10:37:50.868Z level=INFO msg="Upload complete. (496ms)"5417worker2 # [ 114.232819] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:60746 duration_s=0 paths=1 nar_bytes_in=6565418worker2 # [ 114.247311] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:50Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:607465419worker2 # [ 114.253717] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5420worker2 # [ 114.258572] nix-daemon[864]: reaped child process 1392, status = succeeded5421client # building '/nix/store/5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv' on lb:50051...5422worker2 # [ 114.269707] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5423client # farm-b> worker2: building 5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv5424client # farm-b> disabling binary cache 'https://cache.nixos.org' for 60 seconds5425client # farm-b> LOG-farm-b-after-niks3-restart5426worker2 # [ 114.892848] nix-daemon[864]: reaped child process 1406, status = succeeded5427worker2 # [ 115.049543] nix-daemon[864]: accepted connection from pid 1422, user nix-grpc-daemon (trusted)5428worker2 # [ 115.066418] nix-daemon[864]: reaped child process 1425, status = succeeded5429lb # [ 115.061726] niks3-server[1173]: 2026/09/22 10:37:51 INFO Received uploads request method=POST path=/api/pending_closures5430worker2 # [ 115.090356] nix-grpc-daemon[1322]: time=2026-09-22T10:37:51.733Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"5431worker2 # [ 115.091832] nix-grpc-daemon[1322]: time=2026-09-22T10:37:51.733Z level=INFO msg="Uploading myar0ljvsqm9h7fxyh6qrqkg0175r3xn-farm-b (192B)"5432lb # [ 115.103264] niks3-server[1173]: 2026/09/22 10:37:51 INFO Registered completed upload object_key=nar/0zvv5rxmyk9afay83j52m7ipyhrqivhg6h6a0ksjd658pxzwi78w.nar.zst5433lb # [ 115.115885] niks3-server[1173]: 2026/09/22 10:37:51 INFO Registered completed upload object_key=log/5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv5434worker2 # [ 115.143808] nix-grpc-daemon[1322]: time=2026-09-22T10:37:51.787Z level=INFO msg="Uploading 1 narinfos"5435lb # [ 115.125471] niks3-server[1173]: 2026/09/22 10:37:51 INFO Received sign narinfos request method=POST path=/api/pending_closures/32/sign5436lb # [ 115.130934] niks3-server[1173]: 2026/09/22 10:37:51 INFO Signed narinfos id=32 count=15437lb # [ 115.135981] niks3-server[1173]: 2026/09/22 10:37:51 INFO Registered completed upload object_key=myar0ljvsqm9h7fxyh6qrqkg0175r3xn.ls5438lb # [ 115.144178] niks3-server[1173]: 2026/09/22 10:37:51 INFO Received complete upload request method=POST path=/api/pending_closures/32/complete5439lb # [ 115.151333] niks3-server[1173]: 2026/09/22 10:37:51 INFO Completed upload id=325440worker2 # [ 115.169569] nix-grpc-daemon[1322]: time=2026-09-22T10:37:51.813Z level=INFO msg="Upload complete. (288ms)"5441lb # [ 115.154276] niks3-server[1173]: 2026/09/22 10:37:51 INFO Registered completed upload object_key=myar0ljvsqm9h7fxyh6qrqkg0175r3xn.narinfo5442worker2 # [ 115.182124] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:51Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:60746 duration_s=0 drv=5ml8wmsr73y0fh0dflaimhsh2xbcd3ch-farm-b.drv assign_id=2 outputs=15443worker2 # [ 115.200714] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:51Z level=debug event=want client=1 drv=6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv cached=05444worker2 # [ 115.205671] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:51Z level=debug event=assigned drv=6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv worker=192.168.1.4:50051 clients=15445worker2 # [ 115.209923] nix-daemon[864]: reaped child process 1404, status = succeeded5446worker2 # [ 115.238275] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:51Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:607465447worker2 # [ 115.252954] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:51Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:607465448worker2 # [ 115.266805] nix-grpc-daemon[1322]: time=2026-09-22T10:37:51.911Z level=INFO msg="All 2 paths already cached"5449client # copying 1 paths to 'lb:50051'...5450client # copying path '/nix/store/6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv' to 'lb:50051'...5451worker2 # [ 115.277212] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:51Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:607465452worker2 # [ 115.290868] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5453worker2 # [ 115.487670] nix-daemon[864]: accepted connection from pid 1431, user nix-grpc-daemon (trusted)5454worker2 # [ 115.504418] nix-daemon[864]: reaped child process 1434, status = succeeded5455worker2 # [ 115.677987] nix-daemon[864]: accepted connection from pid 1436, user nix-grpc-daemon (trusted)5456worker2 # [ 115.694628] nix-daemon[864]: reaped child process 1439, status = succeeded5457lb # [ 115.687661] niks3-server[1173]: 2026/09/22 10:37:52 INFO Received uploads request method=POST path=/api/pending_closures5458worker2 # [ 115.715735] nix-grpc-daemon[1322]: time=2026-09-22T10:37:52.359Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"5459worker2 # [ 115.717419] nix-grpc-daemon[1322]: time=2026-09-22T10:37:52.359Z level=INFO msg="Uploading 6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv (784B)"5460lb # [ 115.717779] niks3-server[1173]: 2026/09/22 10:37:52 INFO Registered completed upload object_key=nar/1fnvw3c0v6n1j60nd5iagl3bggzcvixsw103w98mwhnpmnmgb248.nar.zst5461lb # [ 115.729192] niks3-server[1173]: 2026/09/22 10:37:52 INFO Received sign narinfos request method=POST path=/api/pending_closures/33/sign5462worker2 # [ 115.753888] nix-grpc-daemon[1322]: time=2026-09-22T10:37:52.398Z level=INFO msg="Uploading 1 narinfos"5463lb # [ 115.740278] niks3-server[1173]: 2026/09/22 10:37:52 INFO Signed narinfos id=33 count=15464lb # [ 115.743181] niks3-server[1173]: 2026/09/22 10:37:52 INFO Registered completed upload object_key=6j7gr603xfslr5lxcbh72430f5q1r4b0.ls5465lb # [ 115.752455] niks3-server[1173]: 2026/09/22 10:37:52 INFO Received complete upload request method=POST path=/api/pending_closures/33/complete5466lb # [ 115.760038] niks3-server[1173]: 2026/09/22 10:37:52 INFO Completed upload id=335467worker2 # [ 115.778183] nix-grpc-daemon[1322]: time=2026-09-22T10:37:52.422Z level=INFO msg="Upload complete. (471ms)"5468lb # [ 115.762539] niks3-server[1173]: 2026/09/22 10:37:52 INFO Registered completed upload object_key=6j7gr603xfslr5lxcbh72430f5q1r4b0.narinfo5469worker2 # [ 115.787172] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:52Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:60746 duration_s=0 paths=1 nar_bytes_in=7845470worker2 # [ 115.798581] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:52Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:607465471worker2 # [ 115.804263] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5472worker2 # [ 115.808742] nix-daemon[864]: reaped child process 1429, status = succeeded5473client # building '/nix/store/6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv' on lb:50051...5474worker2 # [ 115.821390] nix-daemon[864]: accepted connection from pid 1262, user nix-grpc-daemon (trusted)5475client # farm-top> worker2: building 6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv5476client # farm-top> disabling binary cache 'https://cache.nixos.org' for 60 seconds5477client # farm-top> LOG-farm-top-after-niks3-restart5478worker2 # [ 117.044526] nix-daemon[864]: reaped child process 1443, status = succeeded5479worker2 # [ 117.204279] nix-daemon[864]: accepted connection from pid 1459, user nix-grpc-daemon (trusted)5480worker2 # [ 117.221000] nix-daemon[864]: reaped child process 1462, status = succeeded5481lb # [ 117.217356] niks3-server[1173]: 2026/09/22 10:37:53 INFO Received uploads request method=POST path=/api/pending_closures5482worker2 # [ 117.246559] nix-grpc-daemon[1322]: time=2026-09-22T10:37:53.889Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"5483worker2 # [ 117.248094] nix-grpc-daemon[1322]: time=2026-09-22T10:37:53.889Z level=INFO msg="Uploading z8mx9hni3m7cymmz0ddfhg3jhqvi9mlc-farm-top (248B)"5484lb # [ 117.251487] niks3-server[1173]: 2026/09/22 10:37:53 INFO Registered completed upload object_key=log/6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv5485lb # [ 117.263449] niks3-server[1173]: 2026/09/22 10:37:53 INFO Registered completed upload object_key=nar/0cypkbh15calmx90jgh2waxpq661jalf4c99dgb7j606z61xh0ld.nar.zst5486worker2 # [ 117.293097] nix-grpc-daemon[1322]: time=2026-09-22T10:37:53.937Z level=INFO msg="Uploading 1 narinfos"5487lb # [ 117.277904] niks3-server[1173]: 2026/09/22 10:37:53 INFO Received sign narinfos request method=POST path=/api/pending_closures/34/sign5488lb # [ 117.285671] niks3-server[1173]: 2026/09/22 10:37:53 INFO Signed narinfos id=34 count=15489lb # [ 117.287965] niks3-server[1173]: 2026/09/22 10:37:53 INFO Registered completed upload object_key=z8mx9hni3m7cymmz0ddfhg3jhqvi9mlc.ls5490lb # [ 117.297505] niks3-server[1173]: 2026/09/22 10:37:53 INFO Received complete upload request method=POST path=/api/pending_closures/34/complete5491lb # [ 117.302699] niks3-server[1173]: 2026/09/22 10:37:53 INFO Completed upload id=345492worker2 # [ 117.320998] nix-grpc-daemon[1322]: time=2026-09-22T10:37:53.965Z level=INFO msg="Upload complete. (290ms)"5493lb # [ 117.305605] niks3-server[1173]: 2026/09/22 10:37:53 INFO Registered completed upload object_key=z8mx9hni3m7cymmz0ddfhg3jhqvi9mlc.narinfo5494worker2 # [ 117.332898] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:53Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:60746 duration_s=1 drv=6j7gr603xfslr5lxcbh72430f5q1r4b0-farm-top.drv assign_id=3 outputs=15495worker2 # [ 117.347274] nix-daemon[864]: reaped child process 1441, status = succeeded5496client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag after-niks3-restart >&2, in 7.47 seconds)5497client: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag after-niks3-restart outPath5498client: (finished: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag after-niks3-restart outPath, in 0.24 seconds)5499(finished: subtest: a niks3 restart does not cost the leader its role, in 13.20 seconds)5500subtest: scheduler down: the other node takes the lock and keeps it5501lb: must succeed: curl -sf localhost:9901/clusters5502lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5503lb: must succeed: curl -sf localhost:9901/clusters5504lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)5505lb: must succeed: curl -sf localhost:9901/clusters5506lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)5507lb: must succeed: curl -sf localhost:9901/clusters5508lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.04 seconds)5509worker2 # [ 117.936907] systemd[1]: Stopping Nix worker-protocol over gRPC...5510worker2 # [ 118.580245] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:55Z level=info event=signal_exit5511lb # [ 118.808809] niks3-server[1173]: 2026/09/22 10:37:55 INFO lead: released remote=192.168.1.4:431145512lb # [ 119.516382] niks3-server[1173]: 2026/09/22 10:37:56 INFO lead: acquired remote=192.168.1.3:544465513worker1 # [ 119.501968] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:56Z level=info event=scheduler_take_over5514worker1 # [ 119.793879] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:56Z level=info event=scheduler_restarting addr=lb:500515515lb # [ 119.810429] envoy[896]: conn peer=[2001:db8:1::1]:49076 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=9710 tx=4474 ms=71545516worker2 # [ 119.830107] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:56Z level=info event=scheduler_restarting5517worker2 # [ 119.830458] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:56Z level=info event=scheduler_restarting addr=lb:500515518worker2 # [ 119.830652] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:56Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:372125519worker2 # [ 119.830763] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:56Z level=info event=worker_gone addr=192.168.1.4:500515520worker2 # [ 119.848198] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:56Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:372125521worker2 # [ 119.856727] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:56Z level=info event=worker_gone addr=192.168.1.3:500515522worker2 # [ 119.883388] nix-daemon[864]: reaped child process 1314, status = succeeded5523worker1 # [ 119.995358] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:56Z level=info event=scheduler_connected addr=lb:500515524worker1 # [ 120.000669] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:56Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5525worker2 # [ 120.050352] nix-grpc-daemon[1262]: ts=2026-09-22T10:37:56Z level=info event=niks3_push_exited status=succeeded why="niks3 push exited"5526worker1 # [ 120.201478] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:56Z level=info event=scheduler_connected addr=lb:500515527worker1 # [ 120.206856] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:56Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5528worker1 # [ 120.407543] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:57Z level=info event=scheduler_connected addr=lb:500515529worker1 # [ 120.417466] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:57Z level=info event=worker_session_open cn=worker-1 peer=ipv4:192.168.1.2:457805530worker1 # [ 120.422795] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:57Z level=info event=worker_hello addr=192.168.1.3:50051 systems=aarch64-linux max_jobs=1 running=05531worker2 # [ 120.623683] nix-grpc-daemon[1262]: E0000 00:00:1790073477.267840 1307 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 16, started true, shutdown true5532worker2 # [ 120.655583] nix-grpc-daemon[1262]: E0000 00:00:1790073477.299795 1307 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 7, started true, shutdown true5533worker2 # [ 121.768589] systemd[1]: nix-grpc-daemon.service: Deactivated successfully.5534worker2 # [ 121.772811] systemd[1]: Stopped Nix worker-protocol over gRPC.5535worker2 # [ 121.775439] systemd[1]: nix-grpc-daemon.service: Consumed 1.513s CPU time over 22.409s wall clock time, 36.4M memory peak, 45.9K incoming IP traffic, 81.4K outgoing IP traffic.5536worker2 # [ 121.791120] systemd[1]: nix-grpc-daemon.socket: Deactivated successfully.5537worker2 # [ 121.795743] systemd[1]: Closed Nix worker-protocol over gRPC.5538worker2 # [ 121.798888] systemd[1]: nix-grpc-daemon.socket: Consumed 758us CPU time over 22.992s wall clock time, 256K memory peak, 41.9K incoming IP traffic, 32.3K outgoing IP traffic.5539worker1: waiting for success: journalctl -u nix-grpc-daemon | grep -q event=scheduler_take_over5540worker1: (finished: waiting for success: journalctl -u nix-grpc-daemon | grep -q event=scheduler_take_over, in 0.10 seconds)5541worker1: must succeed: curl -sf http://127.0.0.1:9464/metrics5542worker1: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.06 seconds)5543client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag on-standby >&25544worker1 # [ 122.181937] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:457385545client # don't know how to build these paths:5546client # /nix/store/7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv5547worker1 # [ 122.192918] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:457385548worker1 # [ 122.219430] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=debug event=want client=1 drv=adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv cached=05549worker1 # [ 122.226317] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=debug event=assigned drv=adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv worker=192.168.1.3:50051 clients=15550worker1 # [ 122.266835] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385551worker1 # [ 122.285434] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:457385552client # copying 1 paths to 'lb:50051'...5553client # copying path '/nix/store/adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv' to 'lb:50051'...5554worker1 # [ 122.301058] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:457385555worker1 # [ 122.320410] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5556worker1 # [ 122.349162] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:58Z level=info event=niks3_push_started pid=12145557worker1 # [ 122.567052] nix-grpc-daemon[1214]: time=2026-09-22T10:37:59.196Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.key ca=""5558worker1 # [ 122.742314] nix-daemon[844]: accepted connection from pid 1227, user nix-grpc-daemon (trusted)5559worker1 # [ 122.761723] nix-daemon[844]: reaped child process 1230, status = succeeded5560worker1 # [ 122.937328] nix-daemon[844]: accepted connection from pid 1232, user nix-grpc-daemon (trusted)5561worker1 # [ 122.956656] nix-daemon[844]: reaped child process 1235, status = succeeded5562lb # [ 122.989286] niks3-server[1173]: 2026/09/22 10:37:59 INFO Received uploads request method=POST path=/api/pending_closures5563worker1 # [ 122.983469] nix-grpc-daemon[1214]: time=2026-09-22T10:37:59.613Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"5564worker1 # [ 122.985416] nix-grpc-daemon[1214]: time=2026-09-22T10:37:59.613Z level=INFO msg="Uploading adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv (520B)"5565lb # [ 123.025092] niks3-server[1173]: 2026/09/22 10:37:59 INFO Registered completed upload object_key=nar/0scd0jwin78ibwq92wkajw2zj7adk8bih7yhx8bxz1706hh53507.nar.zst5566worker1 # [ 123.029140] nix-grpc-daemon[1214]: time=2026-09-22T10:37:59.659Z level=INFO msg="Uploading 1 narinfos"5567lb # [ 123.039441] niks3-server[1173]: 2026/09/22 10:37:59 INFO Received sign narinfos request method=POST path=/api/pending_closures/35/sign5568lb # [ 123.052940] niks3-server[1173]: 2026/09/22 10:37:59 INFO Signed narinfos id=35 count=15569lb # [ 123.055818] niks3-server[1173]: 2026/09/22 10:37:59 INFO Registered completed upload object_key=adj3g5w8jh83wl959p7daq6igl0gaay3.ls5570lb # [ 123.067678] niks3-server[1173]: 2026/09/22 10:37:59 INFO Received complete upload request method=POST path=/api/pending_closures/35/complete5571worker1 # [ 123.058047] nix-grpc-daemon[1214]: time=2026-09-22T10:37:59.688Z level=INFO msg="Upload complete. (487ms)"5572lb # [ 123.078050] niks3-server[1173]: 2026/09/22 10:37:59 INFO Completed upload id=355573lb # [ 123.081698] niks3-server[1173]: 2026/09/22 10:37:59 INFO Registered completed upload object_key=adj3g5w8jh83wl959p7daq6igl0gaay3.narinfo5574worker1 # [ 123.064941] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:59Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=0 paths=1 nar_bytes_in=5205575worker1 # [ 123.076342] nix-grpc-daemon[1141]: ts=2026-09-22T10:37:59Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385576worker1 # [ 123.089907] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5577worker1 # [ 123.095319] nix-daemon[844]: reaped child process 1212, status = succeeded5578client # building '/nix/store/adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv' on lb:50051...5579worker1 # [ 123.106389] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5580client # farm-a> node-a: building adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv5581client # farm-a> disabling binary cache 'https://cache.nixos.org' for 60 seconds5582client # farm-a> LOG-farm-a-on-standby5583worker1 # [ 124.352437] nix-daemon[844]: reaped child process 1239, status = succeeded5584worker1 # [ 124.524772] nix-daemon[844]: accepted connection from pid 1255, user nix-grpc-daemon (trusted)5585worker1 # [ 124.541616] nix-daemon[844]: reaped child process 1258, status = succeeded5586lb # [ 124.574035] niks3-server[1173]: 2026/09/22 10:38:01 INFO Received uploads request method=POST path=/api/pending_closures5587worker1 # [ 124.568509] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.196Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"5588worker1 # [ 124.570414] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.196Z level=INFO msg="Uploading 7qyaw17jjsjxkkgb1kfm3wxvczk56aan-farm-a (136B)"5589lb # [ 124.605202] niks3-server[1173]: 2026/09/22 10:38:01 INFO Registered completed upload object_key=log/adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv5590lb # [ 124.627423] niks3-server[1173]: 2026/09/22 10:38:01 INFO Registered completed upload object_key=nar/0x76wzgdd1nk8rm930z8l69v9961jpc8qhzmgganv0nhagq8lv9c.nar.zst5591worker1 # [ 124.620248] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.249Z level=INFO msg="Uploading 1 narinfos"5592lb # [ 124.639710] niks3-server[1173]: 2026/09/22 10:38:01 INFO Received sign narinfos request method=POST path=/api/pending_closures/36/sign5593lb # [ 124.650480] niks3-server[1173]: 2026/09/22 10:38:01 INFO Signed narinfos id=36 count=15594lb # [ 124.657312] niks3-server[1173]: 2026/09/22 10:38:01 INFO Registered completed upload object_key=7qyaw17jjsjxkkgb1kfm3wxvczk56aan.ls5595lb # [ 124.669547] niks3-server[1173]: 2026/09/22 10:38:01 INFO Received complete upload request method=POST path=/api/pending_closures/36/complete5596lb # [ 124.676052] niks3-server[1173]: 2026/09/22 10:38:01 INFO Completed upload id=365597worker1 # [ 124.659270] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.290Z level=INFO msg="Upload complete. (337ms)"5598lb # [ 124.679013] niks3-server[1173]: 2026/09/22 10:38:01 INFO Registered completed upload object_key=7qyaw17jjsjxkkgb1kfm3wxvczk56aan.narinfo5599worker1 # [ 124.683579] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=1 drv=adj3g5w8jh83wl959p7daq6igl0gaay3-farm-a.drv assign_id=1 outputs=15600worker1 # [ 124.703228] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=debug event=want client=1 drv=s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv cached=05601worker1 # [ 124.707438] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=debug event=assigned drv=s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv worker=192.168.1.3:50051 clients=15602worker1 # [ 124.711713] nix-daemon[844]: reaped child process 1237, status = succeeded5603worker1 # [ 124.727518] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385604worker1 # [ 124.742677] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:457385605worker1 # [ 124.757587] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.388Z level=INFO msg="All 1 paths already cached"5606client # copying 1 paths to 'lb:50051'...5607client # copying path '/nix/store/s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv' to 'lb:50051'...5608worker1 # [ 124.768136] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:457385609worker1 # [ 124.784195] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5610worker1 # [ 124.983639] nix-daemon[844]: accepted connection from pid 1264, user nix-grpc-daemon (trusted)5611worker1 # [ 125.003125] nix-daemon[844]: reaped child process 1267, status = succeeded5612worker1 # [ 125.177263] nix-daemon[844]: accepted connection from pid 1269, user nix-grpc-daemon (trusted)5613worker1 # [ 125.194004] nix-daemon[844]: reaped child process 1272, status = succeeded5614lb # [ 125.222127] niks3-server[1173]: 2026/09/22 10:38:01 INFO Received uploads request method=POST path=/api/pending_closures5615worker1 # [ 125.215296] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.845Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"5616worker1 # [ 125.216963] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.845Z level=INFO msg="Uploading s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv (640B)"5617lb # [ 125.252584] niks3-server[1173]: 2026/09/22 10:38:01 INFO Registered completed upload object_key=nar/0g5yl4v9pymang94vyp5ib5wi5jwf0bpp184y8qjavkpag12wmlv.nar.zst5618worker1 # [ 125.248930] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.879Z level=INFO msg="Uploading 1 narinfos"5619lb # [ 125.269079] niks3-server[1173]: 2026/09/22 10:38:01 INFO Received sign narinfos request method=POST path=/api/pending_closures/37/sign5620lb # [ 125.279205] niks3-server[1173]: 2026/09/22 10:38:01 INFO Signed narinfos id=37 count=15621worker1 # [ 125.271196] nix-grpc-daemon[1214]: time=2026-09-22T10:38:01.902Z level=INFO msg="Upload complete. (470ms)"5622lb # [ 125.284355] niks3-server[1173]: 2026/09/22 10:38:01 INFO Registered completed upload object_key=s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g.ls5623lb # [ 125.293930] niks3-server[1173]: 2026/09/22 10:38:01 INFO Received complete upload request method=POST path=/api/pending_closures/37/complete5624lb # [ 125.298124] niks3-server[1173]: 2026/09/22 10:38:01 INFO Completed upload id=375625worker1 # [ 125.280236] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=0 paths=1 nar_bytes_in=6405626lb # [ 125.300111] niks3-server[1173]: 2026/09/22 10:38:01 INFO Registered completed upload object_key=s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g.narinfo5627worker1 # [ 125.291169] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:01Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385628worker1 # [ 125.297854] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5629worker1 # [ 125.302490] nix-daemon[844]: reaped child process 1262, status = succeeded5630client # building '/nix/store/s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv' on lb:50051...5631worker1 # [ 125.314879] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5632client # farm-b> node-a: building s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv5633client # farm-b> disabling binary cache 'https://cache.nixos.org' for 60 seconds5634client # farm-b> LOG-farm-b-on-standby5635worker1 # [ 126.536389] nix-daemon[844]: reaped child process 1276, status = succeeded5636worker1 # [ 126.691313] nix-daemon[844]: accepted connection from pid 1292, user nix-grpc-daemon (trusted)5637worker1 # [ 126.710069] nix-daemon[844]: reaped child process 1295, status = succeeded5638lb # [ 126.739389] niks3-server[1173]: 2026/09/22 10:38:03 INFO Received uploads request method=POST path=/api/pending_closures5639worker1 # [ 126.732368] nix-grpc-daemon[1214]: time=2026-09-22T10:38:03.362Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"5640worker1 # [ 126.733814] nix-grpc-daemon[1214]: time=2026-09-22T10:38:03.362Z level=INFO msg="Uploading 6hsxxv65wr2jz8cgv2v66iq16niq1gsv-farm-b (184B)"5641lb # [ 126.774554] niks3-server[1173]: 2026/09/22 10:38:03 INFO Registered completed upload object_key=nar/1grn64wcl2mc78k1jvrz5mcrk1ld70lhf8b9jpiqy8mvs6gfzq0v.nar.zst5642lb # [ 126.789854] niks3-server[1173]: 2026/09/22 10:38:03 INFO Registered completed upload object_key=log/s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv5643lb # [ 126.799326] niks3-server[1173]: 2026/09/22 10:38:03 INFO Received sign narinfos request method=POST path=/api/pending_closures/38/sign5644worker1 # [ 126.788647] nix-grpc-daemon[1214]: time=2026-09-22T10:38:03.419Z level=INFO msg="Uploading 1 narinfos"5645lb # [ 126.809637] niks3-server[1173]: 2026/09/22 10:38:03 INFO Signed narinfos id=38 count=15646lb # [ 126.815161] niks3-server[1173]: 2026/09/22 10:38:03 INFO Registered completed upload object_key=6hsxxv65wr2jz8cgv2v66iq16niq1gsv.ls5647lb # [ 126.822936] niks3-server[1173]: 2026/09/22 10:38:03 INFO Received complete upload request method=POST path=/api/pending_closures/38/complete5648lb # [ 126.831070] niks3-server[1173]: 2026/09/22 10:38:03 INFO Completed upload id=385649worker1 # [ 126.813738] nix-grpc-daemon[1214]: time=2026-09-22T10:38:03.444Z level=INFO msg="Upload complete. (290ms)"5650lb # [ 126.834288] niks3-server[1173]: 2026/09/22 10:38:03 INFO Registered completed upload object_key=6hsxxv65wr2jz8cgv2v66iq16niq1gsv.narinfo5651worker1 # [ 126.830962] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:03Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=1 drv=s5xf4hr9bjv2jdhhqy1ih0c1ld822d2g-farm-b.drv assign_id=2 outputs=15652worker1 # [ 126.846249] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:03Z level=debug event=want client=1 drv=7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv cached=05653worker1 # [ 126.849921] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:03Z level=debug event=assigned drv=7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv worker=192.168.1.3:50051 clients=15654worker1 # [ 126.853574] nix-daemon[844]: reaped child process 1274, status = succeeded5655worker1 # [ 126.875282] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:03Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385656worker1 # [ 126.889579] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:03Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:457385657worker1 # [ 126.903744] nix-grpc-daemon[1214]: time=2026-09-22T10:38:03.534Z level=INFO msg="All 2 paths already cached"5658client # copying 1 paths to 'lb:50051'...5659client # copying path '/nix/store/7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv' to 'lb:50051'...5660worker1 # [ 126.914453] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:03Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:457385661worker1 # [ 126.929404] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5662worker1 # [ 127.127985] nix-daemon[844]: accepted connection from pid 1301, user nix-grpc-daemon (trusted)5663worker1 # [ 127.147712] nix-daemon[844]: reaped child process 1304, status = succeeded5664worker1 # [ 127.321352] nix-daemon[844]: accepted connection from pid 1306, user nix-grpc-daemon (trusted)5665worker1 # [ 127.338023] nix-daemon[844]: reaped child process 1309, status = succeeded5666lb # [ 127.365992] niks3-server[1173]: 2026/09/22 10:38:04 INFO Received uploads request method=POST path=/api/pending_closures5667worker1 # [ 127.357904] nix-grpc-daemon[1214]: time=2026-09-22T10:38:03.988Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"5668worker1 # [ 127.359611] nix-grpc-daemon[1214]: time=2026-09-22T10:38:03.988Z level=INFO msg="Uploading 7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv (768B)"5669lb # [ 127.392613] niks3-server[1173]: 2026/09/22 10:38:04 INFO Registered completed upload object_key=nar/1zyx46mv29cfygdhc5ai40qr4vl88lkr1lilm0h72j23w6wakgmc.nar.zst5670lb # [ 127.404959] niks3-server[1173]: 2026/09/22 10:38:04 INFO Received sign narinfos request method=POST path=/api/pending_closures/39/sign5671worker1 # [ 127.394919] nix-grpc-daemon[1214]: time=2026-09-22T10:38:04.025Z level=INFO msg="Uploading 1 narinfos"5672lb # [ 127.415397] niks3-server[1173]: 2026/09/22 10:38:04 INFO Signed narinfos id=39 count=15673lb # [ 127.424817] niks3-server[1173]: 2026/09/22 10:38:04 INFO Registered completed upload object_key=7s5szig4ffga53df6rls98r9lhxsrsc5.ls5674lb # [ 127.434709] niks3-server[1173]: 2026/09/22 10:38:04 INFO Received complete upload request method=POST path=/api/pending_closures/39/complete5675lb # [ 127.442927] niks3-server[1173]: 2026/09/22 10:38:04 INFO Registered completed upload object_key=7s5szig4ffga53df6rls98r9lhxsrsc5.narinfo5676lb # [ 127.447911] niks3-server[1173]: 2026/09/22 10:38:04 INFO Completed upload id=395677worker1 # [ 127.430717] nix-grpc-daemon[1214]: time=2026-09-22T10:38:04.061Z level=INFO msg="Upload complete. (484ms)"5678worker1 # [ 127.439270] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:04Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=0 paths=1 nar_bytes_in=7685679worker1 # [ 127.448907] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:04Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385680worker1 # [ 127.460229] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5681worker1 # [ 127.464944] nix-daemon[844]: reaped child process 1299, status = succeeded5682client # building '/nix/store/7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv' on lb:50051...5683client # farm-top> node-a: building 7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv5684worker1 # [ 127.478947] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5685client # farm-top> disabling binary cache 'https://cache.nixos.org' for 60 seconds5686client # farm-top> LOG-farm-top-on-standby5687worker1 # [ 129.118428] nix-daemon[844]: reaped child process 1313, status = succeeded5688worker1 # [ 129.298976] nix-daemon[844]: accepted connection from pid 1329, user nix-grpc-daemon (trusted)5689worker1 # [ 129.316648] nix-daemon[844]: reaped child process 1332, status = succeeded5690lb # [ 129.348769] niks3-server[1173]: 2026/09/22 10:38:05 INFO Received uploads request method=POST path=/api/pending_closures5691worker1 # [ 129.341847] nix-grpc-daemon[1214]: time=2026-09-22T10:38:05.971Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"5692worker1 # [ 129.343330] nix-grpc-daemon[1214]: time=2026-09-22T10:38:05.971Z level=INFO msg="Uploading 1ini8mfd1rzqyp8jgqihp4k889588mpx-farm-top (240B)"5693lb # [ 129.381514] niks3-server[1173]: 2026/09/22 10:38:06 INFO Registered completed upload object_key=log/7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv5694lb # [ 129.388221] niks3-server[1173]: 2026/09/22 10:38:06 INFO Registered completed upload object_key=nar/1mncazpqn39nifc2fwbx80la5i264r8xc1lj8fnlbx006znc1925.nar.zst5695lb # [ 129.395508] niks3-server[1173]: 2026/09/22 10:38:06 INFO Received sign narinfos request method=POST path=/api/pending_closures/40/sign5696worker1 # [ 129.382631] nix-grpc-daemon[1214]: time=2026-09-22T10:38:06.013Z level=INFO msg="Uploading 1 narinfos"5697lb # [ 129.402118] niks3-server[1173]: 2026/09/22 10:38:06 INFO Signed narinfos id=40 count=15698lb # [ 129.403681] niks3-server[1173]: 2026/09/22 10:38:06 INFO Registered completed upload object_key=1ini8mfd1rzqyp8jgqihp4k889588mpx.ls5699lb # [ 129.412273] niks3-server[1173]: 2026/09/22 10:38:06 INFO Received complete upload request method=POST path=/api/pending_closures/40/complete5700lb # [ 129.416896] niks3-server[1173]: 2026/09/22 10:38:06 INFO Completed upload id=405701worker1 # [ 129.400070] nix-grpc-daemon[1214]: time=2026-09-22T10:38:06.030Z level=INFO msg="Upload complete. (309ms)"5702lb # [ 129.418651] niks3-server[1173]: 2026/09/22 10:38:06 INFO Registered completed upload object_key=1ini8mfd1rzqyp8jgqihp4k889588mpx.narinfo5703worker1 # [ 129.412923] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:06Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=1 drv=7s5szig4ffga53df6rls98r9lhxsrsc5-farm-top.drv assign_id=3 outputs=15704worker1 # [ 129.424281] nix-daemon[844]: reaped child process 1311, status = succeeded5705client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag on-standby >&2, in 7.54 seconds)5706client: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag on-standby outPath5707client: (finished: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag on-standby outPath, in 0.23 seconds)5708worker2 # [ 129.808821] systemd[1]: Listening on Nix worker-protocol over gRPC.5709worker2 # [ 129.814281] systemd[1]: Starting scheduler-token.service...5710lb # [ 129.810522] envoy[896]: conn peer=[2001:db8:1::4]:43596 tls=TLSv1.3 subject="CN=worker-2" sni=- flags=- tls_fail="-" rx=5496 tx=625 ms=204015711lb # [ 129.815893] envoy[896]: conn peer=[2001:db8:1::1]:39976 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=9679 tx=4430 ms=72415712worker2 # [ 129.909469] systemd[1]: scheduler-token.service: Deactivated successfully.5713worker2 # [ 129.910974] systemd[1]: Finished scheduler-token.service.5714worker2 # [ 129.914683] systemd[1]: scheduler-token.service: Consumed 36ms CPU time over 94ms wall clock time, 2.1M memory peak, 925B incoming IP traffic, 452B outgoing IP traffic.5715worker2 # [ 129.920227] systemd[1]: Starting Nix worker-protocol over gRPC...5716worker2 # [ 130.093776] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:06Z level=info event=warning msg="--niks3 http:// sends the bearer token in clear"5717worker2 # [ 130.095744] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:06Z level=info event=oidc_provider name=mock issuer=http://192.168.1.2:8080/oidc5718worker2 # [ 130.099988] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:06Z level=info event=roles builder=1 scheduler=1 scheduler_addr=lb:50051 advertise=192.168.1.4:50051 niks3=http://lb:5751 max_jobs=15719worker2 # [ 130.107393] nix-grpc-daemon[1483]: WARNING: All log messages before absl::InitializeLog() is called are written to STDERR5720worker2 # [ 130.117272] nix-grpc-daemon[1483]: E0000 00:00:1790073486.751098 1483 tls_security_connector.cc:771] TlsServerCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5721worker2 # [ 130.121192] nix-grpc-daemon[1483]: E0000 00:00:1790073486.754809 1483 tls_security_connector.cc:767] TlsServerCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5722worker2 # [ 130.125330] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:06Z level=info event=startup listen=systemd proxy_socket=/nix/var/nix/daemon-socket/socket5723lb: must succeed: curl -sf localhost:9901/clusters5724worker2 # [ 130.148144] systemd[1]: Started Nix worker-protocol over gRPC.5725worker2 # [ 130.149049] nix-grpc-daemon[1483]: E0000 00:00:1790073486.772255 1525 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5726worker2 # [ 130.151365] nix-grpc-daemon[1483]: E0000 00:00:1790073486.787947 1525 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5727worker2 # [ 130.164826] nix-grpc-daemon[1483]: E0000 00:00:1790073486.788100 1525 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5728worker2 # [ 130.167895] nix-grpc-daemon[1483]: E0000 00:00:1790073486.808222 1525 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5729worker2 # [ 130.181804] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:06Z level=info event=scheduler_connected addr=lb:500515730worker1 # [ 130.159320] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:06Z level=info event=oidc_jwks provider=mock keys=15731worker1 # [ 130.159692] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:06Z level=info event=worker_session_open cn=oidc:mock:node:worker2 peer=ipv4:192.168.1.2:457805732worker1 # [ 130.168924] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:06Z level=info event=worker_hello addr=192.168.1.4:50051 systems=aarch64-linux max_jobs=1 running=05733lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.09 seconds)5734worker1: must succeed: curl -sf http://127.0.0.1:9464/metrics5735worker1: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)5736lb: must succeed: curl -sf localhost:9901/clusters5737lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5738lb: must succeed: curl -sf localhost:9901/clusters5739lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)5740worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics5741worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)5742(finished: subtest: scheduler down: the other node takes the lock and keeps it, in 12.88 seconds)5743subtest: clean scheduler restart: peers are told, running builds survive, no reconnect noise5744client: must succeed: systemd-run --unit rs nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag rs5745client # [ 130.545725] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag rs.5746client # Running as unit: rs.service; invocation ID: b96bc84ad0a641d4adc0c399b399d9fc5747client: (finished: must succeed: systemd-run --unit rs nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag rs, in 0.09 seconds)5748worker2 # [ 130.850325] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:07Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:544745749worker2 # [ 130.861958] nix-daemon[864]: accepted connection from pid 1483, user nix-grpc-daemon (trusted)5750client # [ 130.841905] nix[1294]: don't know how to build these paths:5751client # [ 130.843236] nix[1294]: /nix/store/ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv5752worker1 # [ 130.862565] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:07Z level=debug event=want client=2 drv=ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv cached=05753worker1 # [ 130.862962] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:07Z level=debug event=assigned drv=ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv worker=192.168.1.3:50051 clients=15754worker1 # [ 130.910570] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:07Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385755worker1 # [ 130.925663] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:07Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:457385756client # [ 130.931732] nix[1294]: copying 1 paths to 'lb:50051'...5757client # [ 130.936489] nix[1294]: copying path '/nix/store/ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv' to 'lb:50051'...5758worker1 # [ 130.942707] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:07Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:457385759worker1 # [ 130.957341] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5760worker1 # [ 131.159169] nix-daemon[844]: accepted connection from pid 1345, user nix-grpc-daemon (trusted)5761worker1 # [ 131.177547] nix-daemon[844]: reaped child process 1348, status = succeeded5762worker1 # [ 131.350109] nix-daemon[844]: accepted connection from pid 1350, user nix-grpc-daemon (trusted)5763worker1 # [ 131.365997] nix-daemon[844]: reaped child process 1353, status = succeeded5764lb # [ 131.394516] niks3-server[1173]: 2026/09/22 10:38:08 INFO Received uploads request method=POST path=/api/pending_closures5765worker1 # [ 131.387178] nix-grpc-daemon[1214]: time=2026-09-22T10:38:08.017Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"5766worker1 # [ 131.388750] nix-grpc-daemon[1214]: time=2026-09-22T10:38:08.017Z level=INFO msg="Uploading ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv (416B)"5767lb # [ 131.424416] niks3-server[1173]: 2026/09/22 10:38:08 INFO Received sign narinfos request method=POST path=/api/pending_closures/41/sign5768worker1 # [ 131.416830] nix-grpc-daemon[1214]: time=2026-09-22T10:38:08.047Z level=INFO msg="Uploading 1 narinfos"5769lb # [ 131.440952] niks3-server[1173]: 2026/09/22 10:38:08 INFO Signed narinfos id=41 count=15770lb # [ 131.452848] niks3-server[1173]: 2026/09/22 10:38:08 INFO Registered completed upload object_key=nar/0lynldv3n40qnhi1516wgiry3i3pja3sanwyyd8q4nslr915d7r1.nar.zst5771worker1 # [ 131.453673] nix-grpc-daemon[1214]: time=2026-09-22T10:38:08.084Z level=INFO msg="Upload complete. (479ms)"5772lb # [ 131.470190] niks3-server[1173]: 2026/09/22 10:38:08 INFO Received complete upload request method=POST path=/api/pending_closures/41/complete5773lb # [ 131.479234] niks3-server[1173]: 2026/09/22 10:38:08 INFO Completed upload id=415774worker1 # [ 131.460218] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:08Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=0 paths=1 nar_bytes_in=4165775lb # [ 131.484178] niks3-server[1173]: 2026/09/22 10:38:08 INFO Registered completed upload object_key=ajji8m1zd5q5wi64gp7vclb5dd8hr19m.narinfo5776worker1 # [ 131.473810] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:08Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:457385777worker1 # [ 131.479567] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5778worker1 # [ 131.483969] nix-daemon[844]: reaped child process 1343, status = succeeded5779client # [ 131.487362] nix[1294]: building '/nix/store/ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv' on lb:50051...5780worker1 # [ 131.496366] nix-daemon[844]: accepted connection from pid 1141, user nix-grpc-daemon (trusted)5781lb: must succeed: curl -sf localhost:9901/clusters5782lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5783lb: must succeed: curl -sf localhost:9901/clusters5784lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)5785lb: must succeed: curl -sf localhost:9901/clusters5786lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.05 seconds)5787lb: must succeed: curl -sf localhost:9901/clusters5788lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.04 seconds)5789worker2: must succeed: journalctl --rotate --vacuum-time=1s -u nix-grpc-daemon >/dev/null 2>&1 || true5790worker2 # [ 133.536115] systemd-journald[273]: Received client request to rotate journal, rotating.5791worker2 # [ 133.556675] systemd-journald[273]: Vacuuming done, freed 0B of archived journals from /var/log/journal/ec38f7d974a44b71a6dfde841af12ebf.5792worker2: (finished: must succeed: journalctl --rotate --vacuum-time=1s -u nix-grpc-daemon >/dev/null 2>&1 || true, in 0.10 seconds)5793worker1: must succeed: systemctl restart --no-block nix-grpc-daemon.service5794worker1 # [ 133.582691] systemd[1]: Stopping Nix worker-protocol over gRPC...5795worker1: (finished: must succeed: systemctl restart --no-block nix-grpc-daemon.service, in 0.06 seconds)5796client: must succeed: sleep 5.05797worker1 # [ 134.317238] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:10Z level=info event=draining5798client: (finished: must succeed: sleep 5.0, in 5.06 seconds)5799worker1 # [ 138.922790] nix-daemon[844]: reaped child process 1357, status = succeeded5800worker2: waiting for success: journalctl -u nix-grpc-daemon | grep -q 'event=scheduler_restarting addr='5801worker1 # [ 139.079760] nix-daemon[844]: accepted connection from pid 1398, user nix-grpc-daemon (trusted)5802worker1 # [ 139.100295] nix-daemon[844]: reaped child process 1401, status = succeeded5803lb # [ 139.129426] niks3-server[1173]: 2026/09/22 10:38:15 INFO Received uploads request method=POST path=/api/pending_closures5804worker1 # [ 139.126874] nix-grpc-daemon[1214]: time=2026-09-22T10:38:15.757Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"5805worker1 # [ 139.131042] nix-grpc-daemon[1214]: time=2026-09-22T10:38:15.759Z level=INFO msg="Uploading k48d52qj0czpjmzj15yq3fd14zfw5iq6-slow-rs (120B)"5806lb # [ 139.189731] niks3-server[1173]: 2026/09/22 10:38:15 INFO Registered completed upload object_key=nar/19z3qdigvc4gggg31dgc0wi5w85pgaq9dk7bvngh0yhckpy55jgq.nar.zst5807lb # [ 139.201297] niks3-server[1173]: 2026/09/22 10:38:15 INFO Registered completed upload object_key=log/ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv5808worker1 # [ 139.191193] nix-grpc-daemon[1214]: time=2026-09-22T10:38:15.821Z level=INFO msg="Uploading 1 narinfos"5809lb # [ 139.211987] niks3-server[1173]: 2026/09/22 10:38:15 INFO Received sign narinfos request method=POST path=/api/pending_closures/42/sign5810lb # [ 139.215692] niks3-server[1173]: 2026/09/22 10:38:15 INFO Signed narinfos id=42 count=15811lb # [ 139.230453] niks3-server[1173]: 2026/09/22 10:38:15 INFO Registered completed upload object_key=k48d52qj0czpjmzj15yq3fd14zfw5iq6.ls5812lb # [ 139.234018] niks3-server[1173]: 2026/09/22 10:38:15 INFO Received complete upload request method=POST path=/api/pending_closures/42/complete5813lb # [ 139.238119] niks3-server[1173]: 2026/09/22 10:38:15 INFO Completed upload id=425814worker1 # [ 139.221449] nix-grpc-daemon[1214]: time=2026-09-22T10:38:15.852Z level=INFO msg="Upload complete. (310ms)"5815lb # [ 139.240860] niks3-server[1173]: 2026/09/22 10:38:15 INFO Registered completed upload object_key=k48d52qj0czpjmzj15yq3fd14zfw5iq6.narinfo5816worker1 # [ 139.237222] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:15Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:45738 duration_s=7 drv=ajji8m1zd5q5wi64gp7vclb5dd8hr19m-slow-rs.drv assign_id=4 outputs=15817worker1 # [ 139.253990] nix-daemon[844]: reaped child process 1355, status = succeeded5818client # [ 139.270167] systemd[1]: rs.service: Deactivated successfully.5819client # [ 139.274621] systemd[1]: rs.service: Consumed 186ms CPU time over 8.728s wall clock time, 41.2M memory peak, 7.2K incoming IP traffic, 8.4K outgoing IP traffic.5820worker1 # [ 139.318138] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:15Z level=info event=signal_exit5821lb # [ 139.521244] niks3-server[1173]: 2026/09/22 10:38:16 INFO lead: released remote=192.168.1.3:544465822lb # [ 139.812386] envoy[896]: conn peer=[2001:db8:1::1]:39990 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=2966 tx=1516 ms=84325823lb # [ 140.107048] niks3-server[1173]: 2026/09/22 10:38:16 INFO lead: acquired remote=192.168.1.4:497965824worker2 # [ 140.126161] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:16Z level=info event=scheduler_take_over5825worker2 # [ 140.536898] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:17Z level=info event=scheduler_restarting addr=lb:500515826worker1 # [ 140.508412] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:17Z level=info event=scheduler_restarting5827worker1 # [ 140.512136] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:17Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:457805828worker1 # [ 140.517665] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:17Z level=info event=worker_gone addr=192.168.1.4:500515829worker1 # [ 140.522678] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:17Z level=info event=scheduler_restarting addr=lb:500515830worker1 # [ 140.527113] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:17Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:457805831worker1 # [ 140.533157] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:17Z level=info event=worker_gone addr=192.168.1.3:500515832worker1 # [ 140.555271] nix-daemon[844]: reaped child process 1195, status = succeeded5833worker2 # [ 140.738303] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:17Z level=info event=scheduler_connected addr=lb:500515834worker2 # [ 140.743589] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:17Z level=info event=scheduler_disconnected addr=lb:50051 error="no healthy upstream"5835worker1 # [ 140.728400] nix-grpc-daemon[1141]: ts=2026-09-22T10:38:17Z level=info event=niks3_push_exited status=succeeded why="niks3 push exited"5836worker2 # [ 140.944599] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:17Z level=info event=scheduler_connected addr=lb:500515837worker2 # [ 140.960130] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:17Z level=info event=oidc_jwks provider=mock keys=15838worker2 # [ 140.965661] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:17Z level=info event=worker_session_open cn=oidc:mock:node:worker2 peer=ipv4:192.168.1.2:466345839worker2 # [ 140.971025] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:17Z level=info event=worker_hello addr=192.168.1.4:50051 systems=aarch64-linux max_jobs=1 running=05840worker2: (finished: waiting for success: journalctl -u nix-grpc-daemon | grep -q 'event=scheduler_restarting addr=', in 2.20 seconds)5841client: waiting for success: ! systemctl is-active rs5842client: (finished: waiting for success: ! systemctl is-active rs, in 0.07 seconds)5843client: must succeed: systemctl show -p Result --value rs | grep -qx success || { journalctl -u rs >&2; false; }5844client: (finished: must succeed: systemctl show -p Result --value rs | grep -qx success || { journalctl -u rs >&2; false; }, in 0.05 seconds)5845client: must fail: journalctl -u rs | grep -q 'reconnecting'5846client: (finished: must fail: journalctl -u rs | grep -q 'reconnecting', in 0.04 seconds)5847lb: must succeed: curl -sf localhost:9901/clusters5848lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5849worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics5850worker1 # [ 141.402752] nix-grpc-daemon[1141]: E0000 00:00:1790073498.033399 1186 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 17, started true, shutdown true5851worker1 # [ 141.403335] nix-grpc-daemon[1141]: E0000 00:00:1790073498.033661 1186 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 7, started true, shutdown true5852worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)5853worker1 # [ 142.319708] systemd[1]: nix-grpc-daemon.service: Deactivated successfully.5854worker1 # [ 142.323065] systemd[1]: Stopped Nix worker-protocol over gRPC.5855worker1 # [ 142.328440] systemd[1]: nix-grpc-daemon.service: Consumed 1.808s CPU time over 38.236s wall clock time, 37.2M memory peak, 45.5K incoming IP traffic, 86.6K outgoing IP traffic.5856worker1 # [ 142.343945] systemd[1]: Starting Nix worker-protocol over gRPC...5857lb: must succeed: curl -sf localhost:9901/clusters5858lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5859worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics5860worker1 # [ 142.545235] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:19Z level=info event=warning msg="--niks3 http:// sends the bearer token in clear"5861worker1 # [ 142.547848] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:19Z level=info event=oidc_provider name=mock issuer=http://192.168.1.2:8080/oidc5862worker1 # [ 142.552309] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:19Z level=info event=roles builder=1 scheduler=1 scheduler_addr=lb:50051 advertise=192.168.1.3:50051 niks3=http://lb:5751 max_jobs=15863worker1 # [ 142.560201] nix-grpc-daemon[1408]: WARNING: All log messages before absl::InitializeLog() is called are written to STDERR5864worker1 # [ 142.561632] nix-grpc-daemon[1408]: E0000 00:00:1790073499.189506 1408 tls_security_connector.cc:771] TlsServerCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5865worker1 # [ 142.567706] nix-grpc-daemon[1408]: E0000 00:00:1790073499.196183 1408 tls_security_connector.cc:767] TlsServerCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5866worker1 # [ 142.571943] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:19Z level=info event=startup listen=systemd proxy_socket=/nix/var/nix/daemon-socket/socket5867worker1 # [ 142.574441] systemd[1]: Started Nix worker-protocol over gRPC.5868worker1 # [ 142.580059] nix-grpc-daemon[1408]: E0000 00:00:1790073499.210936 1447 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5869worker1 # [ 142.608157] nix-grpc-daemon[1408]: E0000 00:00:1790073499.228618 1447 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5870worker1 # [ 142.610417] nix-grpc-daemon[1408]: E0000 00:00:1790073499.228783 1447 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5871worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)5872worker1 # [ 142.618173] nix-grpc-daemon[1408]: E0000 00:00:1790073499.248306 1447 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5873worker1 # [ 142.630429] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:19Z level=info event=scheduler_connected addr=lb:500515874worker2 # [ 142.667814] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:19Z level=info event=worker_session_open cn=worker-1 peer=ipv4:192.168.1.2:466345875worker2 # [ 142.672634] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:19Z level=info event=worker_hello addr=192.168.1.3:50051 systems=aarch64-linux max_jobs=1 running=05876lb: must succeed: curl -sf localhost:9901/clusters5877lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5878worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics5879worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)5880(finished: subtest: clean scheduler restart: peers are told, running builds survive, no reconnect noise, in 13.30 seconds)5881subtest: scheduler restart mid-build: build finishes, no double build5882client: must succeed: systemd-run --unit mid nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag mid5883client # [ 143.836993] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag mid.5884client # Running as unit: mid.service; invocation ID: 5db2a217eea64d9199db273cb7bddc2b5885client: (finished: must succeed: systemd-run --unit mid nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag mid, in 0.09 seconds)5886worker1 # [ 144.095042] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:20Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:562145887worker1 # [ 144.110509] nix-daemon[844]: accepted connection from pid 1408, user nix-grpc-daemon (trusted)5888client # [ 144.121817] nix[1330]: don't know how to build these paths:5889client # [ 144.123122] nix[1330]: /nix/store/chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv5890worker2 # [ 144.179557] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:20Z level=debug event=want client=1 drv=chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv cached=05891worker2 # [ 144.185021] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:20Z level=debug event=assigned drv=chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv worker=192.168.1.4:50051 clients=15892worker2 # [ 144.222647] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:20Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:544745893worker2 # [ 144.243422] nix-grpc-daemon[1483]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 26 ms (attempt 1/5)5894worker2 # [ 144.276720] nix-grpc-daemon[1483]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 189 ms (attempt 2/5)5895worker2 # [ 144.473160] nix-grpc-daemon[1483]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 229 ms (attempt 3/5)5896worker2 # [ 144.702825] nix-grpc-daemon[1483]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 289 ms (attempt 4/5)5897worker2 # [ 144.992596] nix-grpc-daemon[1483]: disabling binary cache 'https://cache.nixos.org' for 60 seconds5898worker2 # [ 144.998445] nix-grpc-daemon[1483]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org5899worker2 # [ 145.018434] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:21Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:544745900client # [ 144.992535] nix[1330]: copying 1 paths to 'lb:50051'...5901worker2 # [ 145.034914] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:21Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:544745902client # [ 144.996092] nix[1330]: copying path '/nix/store/chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv' to 'lb:50051'...5903worker2 # [ 145.053079] nix-daemon[864]: accepted connection from pid 1483, user nix-grpc-daemon (trusted)5904worker2 # [ 145.081833] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:21Z level=info event=niks3_push_started pid=16085905worker2 # [ 145.314418] nix-grpc-daemon[1608]: time=2026-09-22T10:38:21.958Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.key ca=""5906worker2 # [ 145.494546] nix-daemon[864]: accepted connection from pid 1625, user nix-grpc-daemon (trusted)5907worker2 # [ 145.520281] nix-daemon[864]: reaped child process 1628, status = succeeded5908worker2 # [ 145.699590] nix-daemon[864]: accepted connection from pid 1630, user nix-grpc-daemon (trusted)5909worker2 # [ 145.717982] nix-daemon[864]: reaped child process 1633, status = succeeded5910lb # [ 145.713979] niks3-server[1173]: 2026/09/22 10:38:22 INFO Received uploads request method=POST path=/api/pending_closures5911worker2 # [ 145.744069] nix-grpc-daemon[1608]: time=2026-09-22T10:38:22.387Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"5912worker2 # [ 145.745519] nix-grpc-daemon[1608]: time=2026-09-22T10:38:22.387Z level=INFO msg="Uploading chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv (424B)"5913lb # [ 145.747981] niks3-server[1173]: 2026/09/22 10:38:22 INFO Registered completed upload object_key=nar/14lxjrdcxwbhc1kgg39y29gnbx5cz80j4852d1i4w3rvq7g4b04g.nar.zst5914lb # [ 145.762810] niks3-server[1173]: 2026/09/22 10:38:22 INFO Received sign narinfos request method=POST path=/api/pending_closures/43/sign5915worker2 # [ 145.786038] nix-grpc-daemon[1608]: time=2026-09-22T10:38:22.430Z level=INFO msg="Uploading 1 narinfos"5916lb # [ 145.769687] niks3-server[1173]: 2026/09/22 10:38:22 INFO Signed narinfos id=43 count=15917lb # [ 145.775846] niks3-server[1173]: 2026/09/22 10:38:22 INFO Registered completed upload object_key=chbb3scb8b4b0sd5wk0a0vcrl8x07pdc.ls5918lb # [ 145.787572] niks3-server[1173]: 2026/09/22 10:38:22 INFO Received complete upload request method=POST path=/api/pending_closures/43/complete5919lb # [ 145.796830] niks3-server[1173]: 2026/09/22 10:38:22 INFO Completed upload id=435920worker2 # [ 145.813868] nix-grpc-daemon[1608]: time=2026-09-22T10:38:22.458Z level=INFO msg="Upload complete. (494ms)"5921lb # [ 145.799090] niks3-server[1173]: 2026/09/22 10:38:22 INFO Registered completed upload object_key=chbb3scb8b4b0sd5wk0a0vcrl8x07pdc.narinfo5922worker2 # [ 145.825841] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:22Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:54474 duration_s=0 paths=1 nar_bytes_in=4245923worker2 # [ 145.837771] nix-grpc-daemon[1483]: ts=2026-09-22T10:38:22Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:544745924worker2 # [ 145.843175] nix-daemon[864]: accepted connection from pid 1483, user nix-grpc-daemon (trusted)5925worker2 # [ 145.847349] nix-daemon[864]: reaped child process 1606, status = succeeded5926client # [ 145.816543] nix[1330]: building '/nix/store/chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv' on lb:50051...5927worker2 # [ 145.859591] nix-daemon[864]: accepted connection from pid 1483, user nix-grpc-daemon (trusted)5928lb: must succeed: curl -sf localhost:9901/clusters5929lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)5930lb: must succeed: curl -sf localhost:9901/clusters5931lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)5932worker2: must succeed: systemctl kill -s KILL nix-grpc-daemon.service; systemctl start nix-grpc-daemon.service5933worker2 # [ 147.862220] systemd[1]: nix-grpc-daemon.service: Sent signal SIGKILL to main process 1483 (nix-grpc-daemon) on client request.5934worker2 # [ 147.864066] systemd[1]: nix-grpc-daemon.service: Killed unit cgroup '/system.slice/nix-grpc-daemon.service' with SIGKILL on client request.5935lb # [ 147.860277] niks3-server[1173]: 2026/09/22 10:38:24 INFO lead: released remote=192.168.1.4:497965936worker2 # [ 147.892090] systemd[1]: Starting scheduler-token.service...5937client # [ 147.851871] nix[1330]: chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv: Stream removed (RST_STREAM (Received RST_STREAM with error code 2)), asking the scheduler again5938worker1 # [ 147.857546] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:24Z level=info event=scheduler_disconnected addr=lb:50051 error="upstream connect error or disconnect/reset before headers. reset reason: connection termination"5939client # [ 147.858113] nix[1330]: scheduler at lb:50051: Stream removed (RST_STREAM (Received RST_STREAM with error code 2)), reconnecting5940worker2 # [ 147.910917] systemd[1]: nix-grpc-daemon.service: Main process exited, code=killed, status=9/KILL5941worker2 # [ 147.920338] systemd[1]: nix-grpc-daemon.service: Failed with result 'signal'.5942worker2 # [ 147.921642] systemd[1]: nix-grpc-daemon.service: Consumed 568ms CPU time over 17.991s wall clock time, 34.4M memory peak, 15.1K incoming IP traffic, 28.3K outgoing IP traffic.5943worker2 # [ 147.933185] nix-daemon[864]: reaped child process 1635, status = succeeded5944worker2 # [ 147.935005] nix-daemon[864]: reaped child process 1542, status = succeeded5945worker2 # [ 147.942126] nix-daemon[1637]: unexpected Nix daemon error: error: interrupted by the user5946worker2 # [ 147.950191] nix-daemon[864]: reaped child process 1637, status = failed with exit code 15947worker2 # [ 148.000388] systemd[1]: scheduler-token.service: Deactivated successfully.5948worker2 # [ 148.001351] systemd[1]: Finished scheduler-token.service.5949worker2 # [ 148.002103] systemd[1]: scheduler-token.service: Consumed 23ms CPU time over 106ms wall clock time, 2.2M memory peak, 925B incoming IP traffic, 452B outgoing IP traffic.5950worker2 # [ 148.003944] systemd[1]: nix-grpc-daemon.service: Scheduled restart job immediately on client request, restart counter is at 1.5951worker2 # [ 148.008534] systemd[1]: Starting scheduler-token.service...5952worker2 # [ 148.078333] systemd[1]: scheduler-token.service: Deactivated successfully.5953worker2 # [ 148.079294] systemd[1]: Finished scheduler-token.service.5954worker2 # [ 148.082623] systemd[1]: scheduler-token.service: Consumed 21ms CPU time over 70ms wall clock time, 2.1M memory peak, 925B incoming IP traffic, 452B outgoing IP traffic.5955worker2 # [ 148.086203] systemd[1]: Starting Nix worker-protocol over gRPC...5956worker1 # [ 148.057721] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:24Z level=info event=scheduler_connected addr=lb:500515957worker2 # [ 148.252562] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=warning msg="--niks3 http:// sends the bearer token in clear"5958worker2 # [ 148.255413] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=oidc_provider name=mock issuer=http://192.168.1.2:8080/oidc5959worker2 # [ 148.259543] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=roles builder=1 scheduler=1 scheduler_addr=lb:50051 advertise=192.168.1.4:50051 niks3=http://lb:5751 max_jobs=15960worker2 # [ 148.268089] nix-grpc-daemon[1680]: WARNING: All log messages before absl::InitializeLog() is called are written to STDERR5961worker2 # [ 148.269559] nix-grpc-daemon[1680]: E0000 00:00:1790073504.910730 1680 tls_security_connector.cc:771] TlsServerCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5962worker2 # [ 148.281382] nix-grpc-daemon[1680]: E0000 00:00:1790073504.916629 1680 tls_security_connector.cc:767] TlsServerCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5963lb # [ 148.267531] niks3-server[1173]: 2026/09/22 10:38:24 INFO lead: acquired remote=192.168.1.4:504765964worker2 # [ 148.287350] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=startup listen=systemd proxy_socket=/nix/var/nix/daemon-socket/socket5965worker2: (finished: must succeed: systemctl kill -s KILL nix-grpc-daemon.service; systemctl start nix-grpc-daemon.service, in 0.48 seconds)5966worker2 # [ 148.291424] systemd[1]: Started Nix worker-protocol over gRPC.5967client: must succeed: sleep 5.05968worker2 # [ 148.292694] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=scheduler_take_over5969worker2 # [ 148.316119] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=worker_session_open cn=worker-1 peer=ipv4:192.168.1.2:466445970worker2 # [ 148.317684] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=worker_hello addr=192.168.1.3:50051 systems=aarch64-linux max_jobs=1 running=05971worker2 # [ 148.319384] nix-grpc-daemon[1680]: E0000 00:00:1790073504.937415 1719 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5972worker2 # [ 148.330720] nix-grpc-daemon[1680]: E0000 00:00:1790073504.953715 1719 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5973worker2 # [ 148.333580] nix-grpc-daemon[1680]: E0000 00:00:1790073504.953885 1719 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.5974worker2 # [ 148.336084] nix-grpc-daemon[1680]: E0000 00:00:1790073504.974212 1719 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.5975worker2 # [ 148.343580] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=scheduler_connected addr=lb:500515976worker2 # [ 148.349213] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=oidc_jwks provider=mock keys=15977worker2 # [ 148.350511] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=worker_session_open cn=oidc:mock:node:worker2 peer=ipv4:192.168.1.2:466445978worker2 # [ 148.352354] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:24Z level=info event=worker_hello addr=192.168.1.4:50051 systems=aarch64-linux max_jobs=1 running=05979worker2 # [ 148.399998] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:25Z level=debug event=want client=1 drv=chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv cached=05980worker2 # [ 148.402452] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:25Z level=debug event=assigned drv=chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv worker=192.168.1.3:50051 clients=15981worker1 # [ 148.411339] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:25Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:562145982worker1 # [ 148.432338] nix-grpc-daemon[1408]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 3 ms (attempt 1/5)5983worker1 # [ 148.442442] nix-grpc-daemon[1408]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 39 ms (attempt 2/5)5984worker1 # [ 148.488861] nix-grpc-daemon[1408]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 290 ms (attempt 3/5)5985worker1 # [ 148.786443] nix-grpc-daemon[1408]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 449 ms (attempt 4/5)5986worker1 # [ 149.244991] nix-grpc-daemon[1408]: disabling binary cache 'https://cache.nixos.org' for 60 seconds5987worker1 # [ 149.250570] nix-grpc-daemon[1408]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org5988worker1 # [ 149.267479] nix-grpc-daemon[1408]: copying path '/nix/store/chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv' from 'http://lb:5751'...5989worker1 # [ 149.296526] nix-daemon[844]: accepted connection from pid 1408, user nix-grpc-daemon (trusted)5990worker1 # [ 149.314592] nix-daemon[844]: accepted connection from pid 1408, user nix-grpc-daemon (trusted)5991client # [ 149.308695] nix[1330]: building '/nix/store/chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv' on lb:50051...5992lb # [ 149.809960] envoy[896]: conn peer=[2001:db8:1::3]:49720 tls=TLSv1.3 subject="CN=worker-1" sni=- flags=- tls_fail="-" rx=1738 tx=673 ms=363485993lb # [ 149.810302] envoy[896]: conn peer=[2001:db8:1::4]:35690 tls=TLSv1.3 subject="CN=worker-2" sni=- flags=- tls_fail="-" rx=2679 tx=399 ms=177215994client: (finished: must succeed: sleep 5.0, in 5.06 seconds)5995client: waiting for success: ! systemctl is-active mid5996worker1 # [ 153.741654] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:30Z level=info event=niks3_push_started pid=15145997worker1 # [ 153.786930] nix-daemon[844]: reaped child process 1489, status = succeeded5998worker1 # [ 153.987257] nix-grpc-daemon[1514]: time=2026-09-22T10:38:30.617Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.key ca=""5999worker1 # [ 154.158888] nix-daemon[844]: accepted connection from pid 1527, user nix-grpc-daemon (trusted)6000worker1 # [ 154.176991] nix-daemon[844]: reaped child process 1530, status = succeeded6001lb # [ 154.209851] niks3-server[1173]: 2026/09/22 10:38:30 INFO Received uploads request method=POST path=/api/pending_closures6002worker1 # [ 154.203254] nix-grpc-daemon[1514]: time=2026-09-22T10:38:30.834Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6003worker1 # [ 154.207911] nix-grpc-daemon[1514]: time=2026-09-22T10:38:30.834Z level=INFO msg="Uploading fk2vdwmn2msb3ja19s2jwak35ignz889-slow-mid (120B)"6004lb # [ 154.256588] niks3-server[1173]: 2026/09/22 10:38:30 INFO Registered completed upload object_key=log/chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv6005worker1 # [ 154.253624] nix-grpc-daemon[1514]: time=2026-09-22T10:38:30.883Z level=INFO msg="Uploading 1 narinfos"6006lb # [ 154.270433] niks3-server[1173]: 2026/09/22 10:38:30 INFO Registered completed upload object_key=nar/0462jb7zwy2d73k3q6v09xnk3bx318v8ik5ckmp32w2hzvwj4k2d.nar.zst6007lb # [ 154.283081] niks3-server[1173]: 2026/09/22 10:38:30 INFO Received sign narinfos request method=POST path=/api/pending_closures/44/sign6008lb # [ 154.289973] niks3-server[1173]: 2026/09/22 10:38:30 INFO Signed narinfos id=44 count=16009lb # [ 154.300125] niks3-server[1173]: 2026/09/22 10:38:30 INFO Registered completed upload object_key=fk2vdwmn2msb3ja19s2jwak35ignz889.ls6010lb # [ 154.303995] niks3-server[1173]: 2026/09/22 10:38:30 INFO Received complete upload request method=POST path=/api/pending_closures/44/complete6011lb # [ 154.314679] niks3-server[1173]: 2026/09/22 10:38:30 INFO Registered completed upload object_key=fk2vdwmn2msb3ja19s2jwak35ignz889.narinfo6012lb # [ 154.329449] niks3-server[1173]: 2026/09/22 10:38:30 INFO Completed upload id=446013worker1 # [ 154.315284] nix-grpc-daemon[1514]: time=2026-09-22T10:38:30.946Z level=INFO msg="Upload complete. (323ms)"6014worker1 # [ 154.328330] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:30Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56214 duration_s=5 drv=chbb3scb8b4b0sd5wk0a0vcrl8x07pdc-slow-mid.drv assign_id=1 outputs=16015worker1 # [ 154.348324] nix-daemon[844]: reaped child process 1487, status = succeeded6016client # [ 154.364885] systemd[1]: mid.service: Deactivated successfully.6017client # [ 154.369143] systemd[1]: mid.service: Consumed 184ms CPU time over 10.531s wall clock time, 39.2M memory peak, 8.7K incoming IP traffic, 10.4K outgoing IP traffic.6018client: (finished: waiting for success: ! systemctl is-active mid, in 1.14 seconds)6019client: must succeed: systemctl show -p Result --value mid | grep -qx success || { journalctl -u mid >&2; false; }6020client: (finished: must succeed: systemctl show -p Result --value mid | grep -qx success || { journalctl -u mid >&2; false; }, in 0.06 seconds)6021(finished: subtest: scheduler restart mid-build: build finishes, no double build, in 11.05 seconds)6022subtest: interrupting the client stops the build on the worker6023client: must succeed: systemd-run --unit intr nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag intr6024client # [ 154.862934] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag intr.6025client # Running as unit: intr.service; invocation ID: 06d6f6c7358948e788d32d6565d4a73e6026client: (finished: must succeed: systemd-run --unit intr nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag intr, in 0.06 seconds)6027worker1 # [ 155.110949] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:31Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:562146028client # [ 155.114478] nix[1366]: don't know how to build these paths:6029client # [ 155.116221] nix[1366]: /nix/store/85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv6030worker2 # [ 155.171516] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:31Z level=debug event=want client=2 drv=85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv cached=06031worker2 # [ 155.177051] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:31Z level=debug event=assigned drv=85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv worker=192.168.1.4:50051 clients=16032worker2 # [ 155.219317] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:31Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646033worker2 # [ 155.234383] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6034worker2 # [ 155.258211] nix-grpc-daemon[1680]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 99 ms (attempt 1/5)6035worker2 # [ 155.363883] nix-grpc-daemon[1680]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 59 ms (attempt 2/5)6036worker2 # [ 155.430596] nix-grpc-daemon[1680]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 147 ms (attempt 3/5)6037worker2 # [ 155.585183] nix-grpc-daemon[1680]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 735 ms (attempt 4/5)6038worker2 # [ 156.329302] nix-grpc-daemon[1680]: disabling binary cache 'https://cache.nixos.org' for 60 seconds6039worker2 # [ 156.334438] nix-grpc-daemon[1680]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org6040worker2 # [ 156.355333] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:32Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646041client # [ 156.328817] nix[1366]: copying 1 paths to 'lb:50051'...6042worker2 # [ 156.372514] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:33Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:561646043client # [ 156.334580] nix[1366]: copying path '/nix/store/85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv' to 'lb:50051'...6044worker2 # [ 156.387837] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6045worker2 # [ 156.415794] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:33Z level=info event=niks3_push_started pid=17606046worker2 # [ 156.625946] nix-grpc-daemon[1760]: time=2026-09-22T10:38:33.269Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker2.key ca=""6047worker2 # [ 156.801008] nix-daemon[864]: accepted connection from pid 1773, user nix-grpc-daemon (trusted)6048worker2 # [ 156.818347] nix-daemon[864]: reaped child process 1776, status = succeeded6049worker2 # [ 156.994402] nix-daemon[864]: accepted connection from pid 1778, user nix-grpc-daemon (trusted)6050worker2 # [ 157.010681] nix-daemon[864]: reaped child process 1781, status = succeeded6051lb # [ 157.005271] niks3-server[1173]: 2026/09/22 10:38:33 INFO Received uploads request method=POST path=/api/pending_closures6052worker2 # [ 157.036065] nix-grpc-daemon[1760]: time=2026-09-22T10:38:33.679Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6053worker2 # [ 157.037534] nix-grpc-daemon[1760]: time=2026-09-22T10:38:33.679Z level=INFO msg="Uploading 85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv (424B)"6054lb # [ 157.047305] niks3-server[1173]: 2026/09/22 10:38:33 INFO Received sign narinfos request method=POST path=/api/pending_closures/45/sign6055worker2 # [ 157.079594] nix-grpc-daemon[1760]: time=2026-09-22T10:38:33.723Z level=INFO msg="Uploading 1 narinfos"6056lb # [ 157.063651] niks3-server[1173]: 2026/09/22 10:38:33 INFO Signed narinfos id=45 count=16057lb # [ 157.071000] niks3-server[1173]: 2026/09/22 10:38:33 INFO Registered completed upload object_key=nar/0h460h76prjyvhqhyxlhz1xrmh3s520jbd19gaagy4hp4vc92r1p.nar.zst6058lb # [ 157.080413] niks3-server[1173]: 2026/09/22 10:38:33 INFO Registered completed upload object_key=85wa5czf6m5ng93xyq44r1affb74csh8.ls6059worker2 # [ 157.107369] nix-grpc-daemon[1760]: time=2026-09-22T10:38:33.751Z level=INFO msg="Upload complete. (477ms)"6060lb # [ 157.089377] niks3-server[1173]: 2026/09/22 10:38:33 INFO Received complete upload request method=POST path=/api/pending_closures/45/complete6061lb # [ 157.095172] niks3-server[1173]: 2026/09/22 10:38:33 INFO Completed upload id=456062lb # [ 157.098745] niks3-server[1173]: 2026/09/22 10:38:33 INFO Registered completed upload object_key=85wa5czf6m5ng93xyq44r1affb74csh8.narinfo6063worker2 # [ 157.116257] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:33Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=0 paths=1 nar_bytes_in=4246064worker2 # [ 157.130483] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:33Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646065worker2 # [ 157.136960] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6066worker2 # [ 157.141602] nix-daemon[864]: reaped child process 1758, status = succeeded6067client # [ 157.110909] nix[1366]: building '/nix/store/85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv' on lb:50051...6068worker2 # [ 157.155372] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6069client: must succeed: systemctl kill -s INT intr6070worker2 # [ 158.624549] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:35Z level=info event=revoked_running drv=85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv6071client # [ 158.581890] systemd[1]: intr.service: Sent signal SIGINT to main process 1366 (nix) on client request.6072client # [ 158.589572] nix[1366]: shutting down85wa5czf6m5ng93xyq44r1affb74csh8-slow-intr.drv: error: gRPC BuildDerivation on 'lb:50051' failed: Stream removed (CANCELLED)6073client: (finished: must succeed: systemctl kill -s INT intr, in 0.08 seconds)6074client # [ 158.602162] nix[1366]: error: interrupted by the user6075client # [ 158.642497] systemd[1]: intr.service: Main process exited, code=exited, status=1/FAILURE6076client # [ 158.648444] systemd[1]: intr.service: Failed with result 'exit-code'.6077client # [ 158.650915] systemd[1]: intr.service: Consumed 166ms CPU time over 3.783s wall clock time, 43.2M memory peak, 6.5K incoming IP traffic, 8.2K outgoing IP traffic.6078worker2 # [ 158.793216] nix-daemon[1785]: unexpected Nix daemon error: error: interrupted by the user6079worker2 # [ 158.799737] nix-daemon[864]: reaped child process 1783, status = succeeded6080worker2 # [ 158.803877] nix-daemon[864]: reaped child process 1785, status = failed with exit code 16081(finished: subtest: interrupting the client stops the build on the worker, in 3.95 seconds)6082subtest: a deploy mid-build drains: the build finishes, the new generation takes over6083client: must succeed: systemd-run --unit sw nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag sw6084client # [ 158.835280] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key" --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag sw.6085client # Running as unit: sw.service; invocation ID: e6d9d58e79444bbd97225316573004016086client: (finished: must succeed: systemd-run --unit sw nix build --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/mpvagzcaddzb5j0jgpk1hnp5dkckqbn5-slow.nix --argstr tag sw, in 0.10 seconds)6087client # [ 159.097494] nix[1387]: don't know how to build these paths:6088client # [ 159.098433] nix[1387]: /nix/store/x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv6089worker1 # [ 159.102556] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:35Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:562146090worker2 # [ 159.148122] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:35Z level=debug event=want client=3 drv=x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv cached=06091worker2 # [ 159.153745] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:35Z level=debug event=assigned drv=x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv worker=192.168.1.3:50051 clients=16092worker1 # [ 159.158864] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:35Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:562146093worker1 # [ 159.174276] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:35Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:562146094client # [ 159.182171] nix[1387]: copying 1 paths to 'lb:50051'...6095worker1 # [ 159.188395] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:35Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:562146096client # [ 159.184614] nix[1387]: copying path '/nix/store/x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv' to 'lb:50051'...6097worker1 # [ 159.205725] nix-daemon[844]: accepted connection from pid 1408, user nix-grpc-daemon (trusted)6098worker1 # [ 159.410473] nix-daemon[844]: accepted connection from pid 1558, user nix-grpc-daemon (trusted)6099worker1 # [ 159.427130] nix-daemon[844]: reaped child process 1561, status = succeeded6100worker1 # [ 159.595664] nix-daemon[844]: accepted connection from pid 1563, user nix-grpc-daemon (trusted)6101worker1 # [ 159.610043] nix-daemon[844]: reaped child process 1566, status = succeeded6102lb # [ 159.645700] niks3-server[1173]: 2026/09/22 10:38:36 INFO Received uploads request method=POST path=/api/pending_closures6103worker1 # [ 159.639072] nix-grpc-daemon[1514]: time=2026-09-22T10:38:36.269Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6104worker1 # [ 159.640634] nix-grpc-daemon[1514]: time=2026-09-22T10:38:36.269Z level=INFO msg="Uploading x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv (416B)"6105lb # [ 159.674528] niks3-server[1173]: 2026/09/22 10:38:36 INFO Registered completed upload object_key=nar/0jczp45bkr8vp1ymznxf6da1iv8dzh42pnl36wiprbzwhpi2ddia.nar.zst6106lb # [ 159.687845] niks3-server[1173]: 2026/09/22 10:38:36 INFO Received sign narinfos request method=POST path=/api/pending_closures/46/sign6107worker1 # [ 159.678766] nix-grpc-daemon[1514]: time=2026-09-22T10:38:36.309Z level=INFO msg="Uploading 1 narinfos"6108lb # [ 159.702986] niks3-server[1173]: 2026/09/22 10:38:36 INFO Signed narinfos id=46 count=16109lb # [ 159.708886] niks3-server[1173]: 2026/09/22 10:38:36 INFO Registered completed upload object_key=x0m4asg5xx7vgj4dip2crwn6zhxqngvs.ls6110lb # [ 159.719108] niks3-server[1173]: 2026/09/22 10:38:36 INFO Received complete upload request method=POST path=/api/pending_closures/46/complete6111worker1 # [ 159.706709] nix-grpc-daemon[1514]: time=2026-09-22T10:38:36.337Z level=INFO msg="Upload complete. (483ms)"6112lb # [ 159.724376] niks3-server[1173]: 2026/09/22 10:38:36 INFO Completed upload id=466113lb # [ 159.728192] niks3-server[1173]: 2026/09/22 10:38:36 INFO Registered completed upload object_key=x0m4asg5xx7vgj4dip2crwn6zhxqngvs.narinfo6114worker1 # [ 159.716322] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:36Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56214 duration_s=0 paths=1 nar_bytes_in=4166115worker1 # [ 159.730523] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:36Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:562146116worker1 # [ 159.740165] nix-daemon[844]: accepted connection from pid 1408, user nix-grpc-daemon (trusted)6117worker1 # [ 159.743168] nix-daemon[844]: reaped child process 1556, status = succeeded6118client # [ 159.747083] nix[1387]: building '/nix/store/x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv' on lb:50051...6119worker1 # [ 159.756603] nix-daemon[844]: accepted connection from pid 1408, user nix-grpc-daemon (trusted)6120lb # [ 159.812619] envoy[896]: conn peer=[2001:db8:1::1]:53444 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=3927 tx=1988 ms=102506121lb # [ 159.817490] envoy[896]: conn peer=[2001:db8:1::1]:44580 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=2974 tx=852 ms=35066122worker1: must succeed: systemctl reload nix-grpc-daemon.service6123worker1 # [ 161.527885] systemd[1]: Reloading Nix worker-protocol over gRPC...6124worker1 # [ 161.579804] systemd[1]: Reloaded Nix worker-protocol over gRPC.6125worker1: (finished: must succeed: systemctl reload nix-grpc-daemon.service, in 0.10 seconds)6126worker1: must succeed: pgrep -f 'read -t [9]0 x'6127worker1: (finished: must succeed: pgrep -f 'read -t [9]0 x', in 0.06 seconds)6128client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag during-drain >&26129client # don't know how to build these paths:6130client # /nix/store/lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv6131worker2 # [ 161.940965] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:38Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:561646132worker2 # [ 161.948603] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:38Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646133worker2 # [ 161.973406] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:38Z level=debug event=want client=4 drv=k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv cached=06134worker2 # [ 161.980158] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:38Z level=debug event=assigned drv=k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv worker=192.168.1.4:50051 clients=16135worker2 # [ 162.021824] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:38Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646136worker2 # [ 162.036486] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:38Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646137client # copying 1 paths to 'lb:50051'...6138worker2 # [ 162.051522] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:38Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:561646139client # copying path '/nix/store/k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv' to 'lb:50051'...6140worker2 # [ 162.075331] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6141worker2 # [ 162.284711] nix-daemon[864]: accepted connection from pid 1836, user nix-grpc-daemon (trusted)6142worker2 # [ 162.303606] nix-daemon[864]: reaped child process 1839, status = succeeded6143worker2 # [ 162.476553] nix-daemon[864]: accepted connection from pid 1841, user nix-grpc-daemon (trusted)6144worker2 # [ 162.491911] nix-daemon[864]: reaped child process 1844, status = succeeded6145lb # [ 162.489472] niks3-server[1173]: 2026/09/22 10:38:39 INFO Received uploads request method=POST path=/api/pending_closures6146worker2 # [ 162.518990] nix-grpc-daemon[1760]: time=2026-09-22T10:38:39.162Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6147worker2 # [ 162.520519] nix-grpc-daemon[1760]: time=2026-09-22T10:38:39.162Z level=INFO msg="Uploading k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv (528B)"6148lb # [ 162.516806] niks3-server[1173]: 2026/09/22 10:38:39 INFO Registered completed upload object_key=nar/1h3fzlmfdwdiqr732wwrq7jnpw7n6p8f9qx1089c6g6x2i4csv75.nar.zst6149lb # [ 162.532383] niks3-server[1173]: 2026/09/22 10:38:39 INFO Received sign narinfos request method=POST path=/api/pending_closures/47/sign6150worker2 # [ 162.556819] nix-grpc-daemon[1760]: time=2026-09-22T10:38:39.201Z level=INFO msg="Uploading 1 narinfos"6151lb # [ 162.544619] niks3-server[1173]: 2026/09/22 10:38:39 INFO Signed narinfos id=47 count=16152lb # [ 162.551615] niks3-server[1173]: 2026/09/22 10:38:39 INFO Registered completed upload object_key=k3l35rf796jrnjns08398p1l8fcx178a.ls6153lb # [ 162.559398] niks3-server[1173]: 2026/09/22 10:38:39 INFO Received complete upload request method=POST path=/api/pending_closures/47/complete6154lb # [ 162.569137] niks3-server[1173]: 2026/09/22 10:38:39 INFO Registered completed upload object_key=k3l35rf796jrnjns08398p1l8fcx178a.narinfo6155lb # [ 162.573079] niks3-server[1173]: 2026/09/22 10:38:39 INFO Completed upload id=476156worker2 # [ 162.590984] nix-grpc-daemon[1760]: time=2026-09-22T10:38:39.235Z level=INFO msg="Upload complete. (498ms)"6157worker2 # [ 162.600454] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:39Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=0 paths=1 nar_bytes_in=5286158worker1 # [ 162.573514] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:39Z level=info event=draining6159worker2 # [ 162.614388] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:39Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646160worker2 # [ 162.621444] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6161worker2 # [ 162.626456] nix-daemon[864]: reaped child process 1834, status = succeeded6162client # building '/nix/store/k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv' on lb:50051...6163client # farm-a> worker2: building k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv6164worker2 # [ 162.639536] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6165client # farm-a> disabling binary cache 'https://cache.nixos.org' for 60 seconds6166client # farm-a> LOG-farm-a-during-drain6167worker2 # [ 163.667755] nix-daemon[864]: reaped child process 1848, status = succeeded6168worker2 # [ 163.828727] nix-daemon[864]: accepted connection from pid 1864, user nix-grpc-daemon (trusted)6169worker2 # [ 163.846637] nix-daemon[864]: reaped child process 1867, status = succeeded6170lb # [ 163.841475] niks3-server[1173]: 2026/09/22 10:38:40 INFO Received uploads request method=POST path=/api/pending_closures6171worker2 # [ 163.870715] nix-grpc-daemon[1760]: time=2026-09-22T10:38:40.513Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6172worker2 # [ 163.874310] nix-grpc-daemon[1760]: time=2026-09-22T10:38:40.513Z level=INFO msg="Uploading 42c24pndjip9nvdj8j25mkv0650griqf-farm-a (136B)"6173lb # [ 163.884345] niks3-server[1173]: 2026/09/22 10:38:40 INFO Registered completed upload object_key=log/k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv6174lb # [ 163.894378] niks3-server[1173]: 2026/09/22 10:38:40 INFO Registered completed upload object_key=nar/0r0px9vw070lid3q4jhnmka11sk2y8kj26ysyvjy9dplavd7mdz4.nar.zst6175lb # [ 163.901946] niks3-server[1173]: 2026/09/22 10:38:40 INFO Received sign narinfos request method=POST path=/api/pending_closures/48/sign6176worker2 # [ 163.926226] nix-grpc-daemon[1760]: time=2026-09-22T10:38:40.569Z level=INFO msg="Uploading 1 narinfos"6177lb # [ 163.908166] niks3-server[1173]: 2026/09/22 10:38:40 INFO Signed narinfos id=48 count=16178lb # [ 163.912844] niks3-server[1173]: 2026/09/22 10:38:40 INFO Registered completed upload object_key=42c24pndjip9nvdj8j25mkv0650griqf.ls6179lb # [ 163.923064] niks3-server[1173]: 2026/09/22 10:38:40 INFO Received complete upload request method=POST path=/api/pending_closures/48/complete6180worker2 # [ 163.947248] nix-grpc-daemon[1760]: time=2026-09-22T10:38:40.591Z level=INFO msg="Upload complete. (298ms)"6181lb # [ 163.930824] niks3-server[1173]: 2026/09/22 10:38:40 INFO Completed upload id=486182lb # [ 163.935165] niks3-server[1173]: 2026/09/22 10:38:40 INFO Registered completed upload object_key=42c24pndjip9nvdj8j25mkv0650griqf.narinfo6183worker2 # [ 163.956831] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:40Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=1 drv=k3l35rf796jrnjns08398p1l8fcx178a-farm-a.drv assign_id=4 outputs=16184worker2 # [ 163.973052] nix-daemon[864]: reaped child process 1846, status = succeeded6185worker2 # [ 163.976703] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:40Z level=debug event=want client=4 drv=im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv cached=06186worker2 # [ 163.981283] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:40Z level=debug event=assigned drv=im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv worker=192.168.1.4:50051 clients=16187worker2 # [ 164.027474] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:40Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646188worker2 # [ 164.043195] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:40Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646189worker2 # [ 164.057447] nix-grpc-daemon[1760]: time=2026-09-22T10:38:40.701Z level=INFO msg="All 1 paths already cached"6190client # copying 1 paths to 'lb:50051'...6191worker2 # [ 164.068121] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:40Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:561646192client # copying path '/nix/store/im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv' to 'lb:50051'...6193worker2 # [ 164.091911] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6194worker2 # [ 164.310228] nix-daemon[864]: accepted connection from pid 1873, user nix-grpc-daemon (trusted)6195worker2 # [ 164.327657] nix-daemon[864]: reaped child process 1876, status = succeeded6196worker2 # [ 164.500060] nix-daemon[864]: accepted connection from pid 1878, user nix-grpc-daemon (trusted)6197worker2 # [ 164.515101] nix-daemon[864]: reaped child process 1881, status = succeeded6198lb # [ 164.511534] niks3-server[1173]: 2026/09/22 10:38:41 INFO Received uploads request method=POST path=/api/pending_closures6199worker2 # [ 164.538915] nix-grpc-daemon[1760]: time=2026-09-22T10:38:41.182Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6200worker2 # [ 164.540497] nix-grpc-daemon[1760]: time=2026-09-22T10:38:41.182Z level=INFO msg="Uploading im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv (640B)"6201lb # [ 164.537386] niks3-server[1173]: 2026/09/22 10:38:41 INFO Registered completed upload object_key=nar/0rxmdnq61ka29n0b2as2hdbz91wi2ap17db890js8r5plhpg436g.nar.zst6202worker2 # [ 164.568814] nix-grpc-daemon[1760]: time=2026-09-22T10:38:41.212Z level=INFO msg="Uploading 1 narinfos"6203lb # [ 164.552990] niks3-server[1173]: 2026/09/22 10:38:41 INFO Received sign narinfos request method=POST path=/api/pending_closures/49/sign6204lb # [ 164.564271] niks3-server[1173]: 2026/09/22 10:38:41 INFO Signed narinfos id=49 count=16205lb # [ 164.569577] niks3-server[1173]: 2026/09/22 10:38:41 INFO Registered completed upload object_key=im7gh7nqqi41f48wvwjqkgv3nm7w5b4l.ls6206worker2 # [ 164.596129] nix-grpc-daemon[1760]: time=2026-09-22T10:38:41.240Z level=INFO msg="Upload complete. (487ms)"6207lb # [ 164.579184] niks3-server[1173]: 2026/09/22 10:38:41 INFO Received complete upload request method=POST path=/api/pending_closures/49/complete6208lb # [ 164.584480] niks3-server[1173]: 2026/09/22 10:38:41 INFO Completed upload id=496209worker2 # [ 164.603361] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:41Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=0 paths=1 nar_bytes_in=6406210lb # [ 164.588691] niks3-server[1173]: 2026/09/22 10:38:41 INFO Registered completed upload object_key=im7gh7nqqi41f48wvwjqkgv3nm7w5b4l.narinfo6211worker2 # [ 164.613847] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:41Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646212worker2 # [ 164.619298] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6213worker2 # [ 164.623887] nix-daemon[864]: reaped child process 1871, status = succeeded6214client # building '/nix/store/im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv' on lb:50051...6215worker2 # [ 164.635706] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6216client # farm-b> worker2: building im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv6217client # farm-b> disabling binary cache 'https://cache.nixos.org' for 60 seconds6218client # farm-b> LOG-farm-b-during-drain6219worker2 # [ 165.460435] nix-daemon[864]: reaped child process 1885, status = succeeded6220worker2 # [ 165.619774] nix-daemon[864]: accepted connection from pid 1901, user nix-grpc-daemon (trusted)6221worker2 # [ 165.634725] nix-daemon[864]: reaped child process 1904, status = succeeded6222lb # [ 165.633312] niks3-server[1173]: 2026/09/22 10:38:42 INFO Received uploads request method=POST path=/api/pending_closures6223worker2 # [ 165.662348] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.305Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6224worker2 # [ 165.663781] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.305Z level=INFO msg="Uploading daks7y2c761g04qpzsjx25629wigr8js-farm-b (184B)"6225lb # [ 165.664598] niks3-server[1173]: 2026/09/22 10:38:42 INFO Registered completed upload object_key=log/im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv6226lb # [ 165.677693] niks3-server[1173]: 2026/09/22 10:38:42 INFO Registered completed upload object_key=nar/0lnpxl34nrzp147jy5bafczkvxcl00hcf69qjhsljmv2ip9jisfh.nar.zst6227worker2 # [ 165.707960] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.352Z level=INFO msg="Uploading 1 narinfos"6228lb # [ 165.691404] niks3-server[1173]: 2026/09/22 10:38:42 INFO Received sign narinfos request method=POST path=/api/pending_closures/50/sign6229lb # [ 165.699978] niks3-server[1173]: 2026/09/22 10:38:42 INFO Signed narinfos id=50 count=16230lb # [ 165.708269] niks3-server[1173]: 2026/09/22 10:38:42 INFO Registered completed upload object_key=daks7y2c761g04qpzsjx25629wigr8js.ls6231lb # [ 165.711705] niks3-server[1173]: 2026/09/22 10:38:42 INFO Received complete upload request method=POST path=/api/pending_closures/50/complete6232lb # [ 165.716033] niks3-server[1173]: 2026/09/22 10:38:42 INFO Completed upload id=506233worker2 # [ 165.733814] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.378Z level=INFO msg="Upload complete. (289ms)"6234lb # [ 165.718591] niks3-server[1173]: 2026/09/22 10:38:42 INFO Registered completed upload object_key=daks7y2c761g04qpzsjx25629wigr8js.narinfo6235worker2 # [ 165.749496] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:42Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=1 drv=im7gh7nqqi41f48wvwjqkgv3nm7w5b4l-farm-b.drv assign_id=5 outputs=16236worker2 # [ 165.763575] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:42Z level=debug event=want client=4 drv=lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv cached=06237worker2 # [ 165.767272] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:42Z level=debug event=assigned drv=lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv worker=192.168.1.4:50051 clients=16238worker2 # [ 165.770915] nix-daemon[864]: reaped child process 1883, status = succeeded6239worker2 # [ 165.794121] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:42Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646240worker2 # [ 165.807730] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:42Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646241worker2 # [ 165.821646] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.465Z level=INFO msg="All 2 paths already cached"6242client # copying 1 paths to 'lb:50051'...6243client # copying path '/nix/store/lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv' to 'lb:50051'...6244worker2 # [ 165.832690] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:42Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:561646245worker2 # [ 165.850349] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6246worker2 # [ 166.054047] nix-daemon[864]: accepted connection from pid 1910, user nix-grpc-daemon (trusted)6247worker2 # [ 166.072226] nix-daemon[864]: reaped child process 1913, status = succeeded6248worker2 # [ 166.247530] nix-daemon[864]: accepted connection from pid 1915, user nix-grpc-daemon (trusted)6249worker2 # [ 166.263491] nix-daemon[864]: reaped child process 1918, status = succeeded6250lb # [ 166.258958] niks3-server[1173]: 2026/09/22 10:38:42 INFO Received uploads request method=POST path=/api/pending_closures6251worker2 # [ 166.286767] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.930Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6252worker2 # [ 166.288424] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.930Z level=INFO msg="Uploading lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv (768B)"6253lb # [ 166.285663] niks3-server[1173]: 2026/09/22 10:38:42 INFO Registered completed upload object_key=nar/066jhl91shf8ar6h3cqqb855j5cswl7170wrg5lzf3fhpka0xywn.nar.zst6254lb # [ 166.299034] niks3-server[1173]: 2026/09/22 10:38:42 INFO Received sign narinfos request method=POST path=/api/pending_closures/51/sign6255worker2 # [ 166.324484] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.968Z level=INFO msg="Uploading 1 narinfos"6256lb # [ 166.309933] niks3-server[1173]: 2026/09/22 10:38:42 INFO Signed narinfos id=51 count=16257lb # [ 166.316636] niks3-server[1173]: 2026/09/22 10:38:42 INFO Registered completed upload object_key=lbk8wy49c881163x07gb8z669j2fg1ff.ls6258lb # [ 166.326005] niks3-server[1173]: 2026/09/22 10:38:42 INFO Received complete upload request method=POST path=/api/pending_closures/51/complete6259worker2 # [ 166.349328] nix-grpc-daemon[1760]: time=2026-09-22T10:38:42.993Z level=INFO msg="Upload complete. (480ms)"6260lb # [ 166.331141] niks3-server[1173]: 2026/09/22 10:38:42 INFO Completed upload id=516261lb # [ 166.334215] niks3-server[1173]: 2026/09/22 10:38:42 INFO Registered completed upload object_key=lbk8wy49c881163x07gb8z669j2fg1ff.narinfo6262worker2 # [ 166.356142] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:42Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=0 paths=1 nar_bytes_in=7686263worker2 # [ 166.368744] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:43Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646264worker2 # [ 166.374497] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6265worker2 # [ 166.379209] nix-daemon[864]: reaped child process 1908, status = succeeded6266client # building '/nix/store/lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv' on lb:50051...6267worker2 # [ 166.392992] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6268client # farm-top> worker2: building lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv6269client # farm-top> disabling binary cache 'https://cache.nixos.org' for 60 seconds6270client # farm-top> LOG-farm-top-during-drain6271worker2 # [ 167.218772] nix-daemon[864]: reaped child process 1922, status = succeeded6272worker2 # [ 167.387236] nix-daemon[864]: accepted connection from pid 1938, user nix-grpc-daemon (trusted)6273worker2 # [ 167.403983] nix-daemon[864]: reaped child process 1941, status = succeeded6274lb # [ 167.398823] niks3-server[1173]: 2026/09/22 10:38:44 INFO Received uploads request method=POST path=/api/pending_closures6275worker2 # [ 167.428502] nix-grpc-daemon[1760]: time=2026-09-22T10:38:44.071Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6276worker2 # [ 167.431178] nix-grpc-daemon[1760]: time=2026-09-22T10:38:44.071Z level=INFO msg="Uploading xdwmx2f4n5n8bggglniz2n35any0xz4w-farm-top (240B)"6277lb # [ 167.430503] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=log/lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv6278lb # [ 167.446537] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=nar/1bpy9ga4n2446pary6msidbs8mnil1a34jn8p8yl2scvdg691mz2.nar.zst6279lb # [ 167.457447] niks3-server[1173]: 2026/09/22 10:38:44 INFO Received sign narinfos request method=POST path=/api/pending_closures/52/sign6280worker2 # [ 167.480965] nix-grpc-daemon[1760]: time=2026-09-22T10:38:44.124Z level=INFO msg="Uploading 1 narinfos"6281lb # [ 167.465636] niks3-server[1173]: 2026/09/22 10:38:44 INFO Signed narinfos id=52 count=16282lb # [ 167.470678] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=xdwmx2f4n5n8bggglniz2n35any0xz4w.ls6283lb # [ 167.480570] niks3-server[1173]: 2026/09/22 10:38:44 INFO Received complete upload request method=POST path=/api/pending_closures/52/complete6284lb # [ 167.486986] niks3-server[1173]: 2026/09/22 10:38:44 INFO Completed upload id=526285worker2 # [ 167.504412] nix-grpc-daemon[1760]: time=2026-09-22T10:38:44.148Z level=INFO msg="Upload complete. (302ms)"6286lb # [ 167.490098] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=xdwmx2f4n5n8bggglniz2n35any0xz4w.narinfo6287worker2 # [ 167.516352] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:44Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=1 drv=lbk8wy49c881163x07gb8z669j2fg1ff-farm-top.drv assign_id=6 outputs=16288worker2 # [ 167.532604] nix-daemon[864]: reaped child process 1920, status = succeeded6289client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag during-drain >&2, in 5.89 seconds)6290client: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag during-drain outPath6291client: (finished: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag during-drain outPath, in 0.23 seconds)6292worker1: must succeed: pkill -f 'read -t [9]0 x'6293worker1: (finished: must succeed: pkill -f 'read -t [9]0 x', in 0.06 seconds)6294client: waiting for success: ! systemctl is-active sw6295worker1 # [ 168.011689] nix-daemon[844]: reaped child process 1570, status = succeeded6296worker1 # [ 168.185003] nix-daemon[844]: accepted connection from pid 1611, user nix-grpc-daemon (trusted)6297worker1 # [ 168.201884] nix-daemon[844]: reaped child process 1614, status = succeeded6298lb # [ 168.230210] niks3-server[1173]: 2026/09/22 10:38:44 INFO Received uploads request method=POST path=/api/pending_closures6299worker1 # [ 168.223247] nix-grpc-daemon[1514]: time=2026-09-22T10:38:44.853Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6300worker1 # [ 168.224899] nix-grpc-daemon[1514]: time=2026-09-22T10:38:44.853Z level=INFO msg="Uploading qr8vyl2zl3nxnin3vw41c6z35vj7087f-slow-sw (120B)"6301lb # [ 168.261513] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=nar/0vi6cx31wc05wns04pr5vr2bak9iib1idndvy6gik6c2rz9q8c19.nar.zst6302lb # [ 168.280448] niks3-server[1173]: 2026/09/22 10:38:44 INFO Received sign narinfos request method=POST path=/api/pending_closures/53/sign6303worker1 # [ 168.273376] nix-grpc-daemon[1514]: time=2026-09-22T10:38:44.902Z level=INFO msg="Uploading 1 narinfos"6304lb # [ 168.296858] niks3-server[1173]: 2026/09/22 10:38:44 INFO Signed narinfos id=53 count=16305lb # [ 168.298989] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=log/x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv6306lb # [ 168.313372] niks3-server[1173]: 2026/09/22 10:38:44 INFO Received complete upload request method=POST path=/api/pending_closures/53/complete6307lb # [ 168.315999] niks3-server[1173]: 2026/09/22 10:38:44 INFO Completed upload id=536308worker1 # [ 168.294967] nix-grpc-daemon[1514]: time=2026-09-22T10:38:44.925Z level=INFO msg="Upload complete. (320ms)"6309lb # [ 168.323604] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=qr8vyl2zl3nxnin3vw41c6z35vj7087f.narinfo6310lb # [ 168.330496] niks3-server[1173]: 2026/09/22 10:38:44 INFO Registered completed upload object_key=qr8vyl2zl3nxnin3vw41c6z35vj7087f.ls6311worker1 # [ 168.309396] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:44Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56214 duration_s=8 drv=x0m4asg5xx7vgj4dip2crwn6zhxqngvs-slow-sw.drv assign_id=3 outputs=16312worker1 # [ 168.328303] nix-daemon[844]: reaped child process 1568, status = succeeded6313client # [ 168.342438] systemd[1]: sw.service: Deactivated successfully.6314client # [ 168.347714] systemd[1]: sw.service: Consumed 175ms CPU time over 9.510s wall clock time, 41.2M memory peak, 7.3K incoming IP traffic, 8.4K outgoing IP traffic.6315worker1 # [ 168.574087] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:45Z level=info event=signal_exit6316client: (finished: waiting for success: ! systemctl is-active sw, in 1.12 seconds)6317client: must succeed: systemctl show -p Result --value sw | grep -qx success || { journalctl -u sw >&2; false; }6318client: (finished: must succeed: systemctl show -p Result --value sw | grep -qx success || { journalctl -u sw >&2; false; }, in 0.06 seconds)6319worker1: waiting for success: systemctl is-active nix-grpc-daemon.service || systemctl start nix-grpc-daemon.service6320lb # [ 169.811544] envoy[896]: conn peer=[2001:db8:1::1]:59574 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=9608 tx=4427 ms=55916321lb # [ 169.816674] envoy[896]: conn peer=[2001:db8:1::1]:44582 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=2978 tx=1541 ms=92206322worker1 # [ 169.804221] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:46Z level=info event=scheduler_restarting6323worker2 # [ 169.849431] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:46Z level=info event=worker_session_closed peer=ipv4:192.168.1.2:466446324worker2 # [ 169.854066] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:46Z level=info event=worker_gone addr=192.168.1.3:500516325worker1 # [ 169.833856] nix-grpc-daemon[1408]: ts=2026-09-22T10:38:46Z level=info event=niks3_push_exited status=succeeded why="niks3 push exited"6326worker1 # [ 169.859061] nix-daemon[844]: reaped child process 1464, status = succeeded6327worker1 # [ 170.358777] nix-grpc-daemon[1408]: E0000 00:00:1790073526.989437 1453 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 15, started true, shutdown true6328worker1 # [ 170.365408] nix-grpc-daemon[1408]: E0000 00:00:1790073526.989674 1453 external_connection_acceptor_impl.cc:72] NOT handling external connection with fd 7, started true, shutdown true6329worker1 # [ 170.611268] systemd[1]: nix-grpc-daemon.service: Deactivated successfully.6330worker1 # [ 170.619006] systemd[1]: nix-grpc-daemon.service: Consumed 896ms CPU time over 28.267s wall clock time, 33.5M memory peak, 20.8K incoming IP traffic, 27.2K outgoing IP traffic.6331worker1 # [ 170.638673] systemd[1]: Starting Nix worker-protocol over gRPC...6332worker1 # [ 170.837757] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:47Z level=info event=warning msg="--niks3 http:// sends the bearer token in clear"6333worker1 # [ 170.840386] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:47Z level=info event=oidc_provider name=mock issuer=http://192.168.1.2:8080/oidc6334worker1 # [ 170.844458] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:47Z level=info event=roles builder=1 scheduler=1 scheduler_addr=lb:50051 advertise=192.168.1.3:50051 niks3=http://lb:5751 max_jobs=16335worker1 # [ 170.852064] nix-grpc-daemon[1628]: WARNING: All log messages before absl::InitializeLog() is called are written to STDERR6336worker1 # [ 170.853489] nix-grpc-daemon[1628]: E0000 00:00:1790073527.481678 1628 tls_security_connector.cc:771] TlsServerCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.6337worker1 # [ 170.862028] nix-grpc-daemon[1628]: E0000 00:00:1790073527.487866 1628 tls_security_connector.cc:767] TlsServerCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.6338worker1 # [ 170.866169] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:47Z level=info event=startup listen=systemd proxy_socket=/nix/var/nix/daemon-socket/socket6339worker1 # [ 170.869651] systemd[1]: Started Nix worker-protocol over gRPC.6340worker1 # [ 170.876094] nix-grpc-daemon[1628]: E0000 00:00:1790073527.503754 1668 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.6341worker1: (finished: waiting for success: systemctl is-active nix-grpc-daemon.service || systemctl start nix-grpc-daemon.service, in 1.89 seconds)6342lb: waiting until exactly no endpoints of aarch64-linux are unhealthy6343lb: must succeed: curl -sf localhost:9901/clusters6344worker1 # [ 170.904053] nix-grpc-daemon[1628]: E0000 00:00:1790073527.521598 1668 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.6345worker1 # [ 170.906333] nix-grpc-daemon[1628]: E0000 00:00:1790073527.522491 1668 tls_security_connector.cc:493] TlsChannelCertificateWatcher getting identity_cert_error: UNKNOWN:Unable to get latest identity certificates.6346worker1 # [ 170.911797] nix-grpc-daemon[1628]: E0000 00:00:1790073527.541812 1668 tls_security_connector.cc:489] TlsChannelCertificateWatcher getting root_cert_error: UNKNOWN:Unable to get latest root certificates.6347worker1 # [ 170.928629] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:47Z level=info event=scheduler_connected addr=lb:500516348worker2 # [ 170.966266] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:47Z level=info event=worker_session_open cn=worker-1 peer=ipv4:192.168.1.2:466446349worker2 # [ 170.966643] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:47Z level=info event=worker_hello addr=192.168.1.3:50051 systems=aarch64-linux max_jobs=1 running=06350lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.09 seconds)6351lb: must succeed: curl -sf localhost:9901/clusters6352lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)6353lb: (finished: waiting until exactly no endpoints of aarch64-linux are unhealthy, in 1.17 seconds)6354lb: must succeed: curl -sf localhost:9901/clusters6355lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)6356worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics6357worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)6358(finished: subtest: a deploy mid-build drains: the build finishes, the new generation takes over, in 13.43 seconds)6359subtest: requiredSystemFeatures: placed on the worker that has them, refused when none does6360client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto --expr 'map (tag: import /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix { inherit tag; features = ["vip"]; }) ["f1" "f2" "f3"]' --impure 2>&16361worker2 # [ 172.503699] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:49Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:561646362worker1 # [ 172.477327] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:49Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546363worker1 # [ 172.493035] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6364worker1 # [ 172.517626] nix-grpc-daemon[1628]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 58 ms (attempt 1/5)6365worker1 # [ 172.583270] nix-grpc-daemon[1628]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 4 ms (attempt 2/5)6366worker1 # [ 172.594581] nix-grpc-daemon[1628]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 266 ms (attempt 3/5)6367worker1 # [ 172.867915] nix-grpc-daemon[1628]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org; retrying in 226 ms (attempt 4/5)6368worker1 # [ 173.102165] nix-grpc-daemon[1628]: disabling binary cache 'https://cache.nixos.org' for 60 seconds6369worker1 # [ 173.108096] nix-grpc-daemon[1628]: warning: unable to download 'https://cache.nixos.org/nix-cache-info': Could not resolve hostname (6) Could not resolve host: cache.nixos.org6370worker2 # [ 173.189604] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:49Z level=debug event=want client=5 drv=3223c4sl4ncpldbvw558vqwddijwzfpd-farm-a.drv cached=06371worker2 # [ 173.190010] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:49Z level=debug event=assigned drv=3223c4sl4ncpldbvw558vqwddijwzfpd-farm-a.drv worker=192.168.1.3:50051 clients=16372worker2 # [ 173.190145] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:49Z level=debug event=want client=5 drv=pplpzcnjfn295nm16cg0iihpfc9iffry-farm-a.drv cached=06373worker2 # [ 173.190272] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:49Z level=debug event=want client=5 drv=sy43wp9v52pni1wms6q0p3qybpc2pwad-farm-a.drv cached=06374worker1 # [ 173.199341] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:49Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546375worker1 # [ 173.215774] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:49Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546376worker1 # [ 173.230257] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:49Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546377worker1 # [ 173.244684] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6378worker1 # [ 173.274887] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:49Z level=info event=niks3_push_started pid=16886379worker1 # [ 173.493930] nix-grpc-daemon[1688]: time=2026-09-22T10:38:50.124Z level=INFO msg="Configuring client TLS" cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.pem key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/worker1.key ca=""6380worker1 # [ 173.670522] nix-daemon[844]: accepted connection from pid 1701, user nix-grpc-daemon (trusted)6381worker1 # [ 173.687755] nix-daemon[844]: reaped child process 1704, status = succeeded6382worker1 # [ 173.871694] nix-daemon[844]: accepted connection from pid 1706, user nix-grpc-daemon (trusted)6383worker1 # [ 173.886696] nix-daemon[844]: reaped child process 1709, status = succeeded6384lb # [ 173.921539] niks3-server[1173]: 2026/09/22 10:38:50 INFO Received uploads request method=POST path=/api/pending_closures6385worker1 # [ 173.915562] nix-grpc-daemon[1688]: time=2026-09-22T10:38:50.545Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6386worker1 # [ 173.917437] nix-grpc-daemon[1688]: time=2026-09-22T10:38:50.545Z level=INFO msg="Uploading 3223c4sl4ncpldbvw558vqwddijwzfpd-farm-a.drv (512B)"6387lb # [ 173.954495] niks3-server[1173]: 2026/09/22 10:38:50 INFO Registered completed upload object_key=nar/0zxfsmqijfc2dkglls3yp1x3w0w1na7vx39fxc124zddjzpaygd8.nar.zst6388lb # [ 173.966869] niks3-server[1173]: 2026/09/22 10:38:50 INFO Received sign narinfos request method=POST path=/api/pending_closures/54/sign6389worker1 # [ 173.956759] nix-grpc-daemon[1688]: time=2026-09-22T10:38:50.587Z level=INFO msg="Uploading 1 narinfos"6390lb # [ 173.979429] niks3-server[1173]: 2026/09/22 10:38:50 INFO Signed narinfos id=54 count=16391lb # [ 173.986410] niks3-server[1173]: 2026/09/22 10:38:50 INFO Registered completed upload object_key=3223c4sl4ncpldbvw558vqwddijwzfpd.ls6392lb # [ 173.999468] niks3-server[1173]: 2026/09/22 10:38:50 INFO Received complete upload request method=POST path=/api/pending_closures/54/complete6393worker1 # [ 173.989149] nix-grpc-daemon[1688]: time=2026-09-22T10:38:50.620Z level=INFO msg="Upload complete. (490ms)"6394lb # [ 174.007469] niks3-server[1173]: 2026/09/22 10:38:50 INFO Completed upload id=546395lb # [ 174.011552] niks3-server[1173]: 2026/09/22 10:38:50 INFO Registered completed upload object_key=3223c4sl4ncpldbvw558vqwddijwzfpd.narinfo6396worker1 # [ 173.996729] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:50Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=5126397worker1 # [ 174.011114] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:50Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546398worker1 # [ 174.020222] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6399worker1 # [ 174.023282] nix-daemon[844]: reaped child process 1686, status = succeeded6400worker1 # [ 174.034261] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6401worker1 # [ 175.084798] nix-daemon[844]: reaped child process 1713, status = succeeded6402worker1 # [ 175.267805] nix-daemon[844]: accepted connection from pid 1729, user nix-grpc-daemon (trusted)6403worker1 # [ 175.284507] nix-daemon[844]: reaped child process 1732, status = succeeded6404lb # [ 175.317944] niks3-server[1173]: 2026/09/22 10:38:51 INFO Received uploads request method=POST path=/api/pending_closures6405worker1 # [ 175.311064] nix-grpc-daemon[1688]: time=2026-09-22T10:38:51.938Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6406worker1 # [ 175.313266] nix-grpc-daemon[1688]: time=2026-09-22T10:38:51.938Z level=INFO msg="Uploading dla3w0zf02z5jhccwa27i85bbgzrrsvb-farm-a (128B)"6407lb # [ 175.344856] niks3-server[1173]: 2026/09/22 10:38:51 INFO Registered completed upload object_key=log/3223c4sl4ncpldbvw558vqwddijwzfpd-farm-a.drv6408lb # [ 175.359846] niks3-server[1173]: 2026/09/22 10:38:51 INFO Registered completed upload object_key=nar/05i7ka1qa7vmhiys96ln3pp3nj3rfdlslnbydrxdqj40cmvmqcwf.nar.zst6409lb # [ 175.374936] niks3-server[1173]: 2026/09/22 10:38:52 INFO Received sign narinfos request method=POST path=/api/pending_closures/55/sign6410worker1 # [ 175.366162] nix-grpc-daemon[1688]: time=2026-09-22T10:38:51.995Z level=INFO msg="Uploading 1 narinfos"6411lb # [ 175.388100] niks3-server[1173]: 2026/09/22 10:38:52 INFO Signed narinfos id=55 count=16412lb # [ 175.394216] niks3-server[1173]: 2026/09/22 10:38:52 INFO Registered completed upload object_key=dla3w0zf02z5jhccwa27i85bbgzrrsvb.ls6413lb # [ 175.403941] niks3-server[1173]: 2026/09/22 10:38:52 INFO Received complete upload request method=POST path=/api/pending_closures/55/complete6414lb # [ 175.409104] niks3-server[1173]: 2026/09/22 10:38:52 INFO Completed upload id=556415worker1 # [ 175.391892] nix-grpc-daemon[1688]: time=2026-09-22T10:38:52.022Z level=INFO msg="Upload complete. (345ms)"6416lb # [ 175.412128] niks3-server[1173]: 2026/09/22 10:38:52 INFO Registered completed upload object_key=dla3w0zf02z5jhccwa27i85bbgzrrsvb.narinfo6417worker2 # [ 175.437284] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:52Z level=debug event=assigned drv=pplpzcnjfn295nm16cg0iihpfc9iffry-farm-a.drv worker=192.168.1.3:50051 clients=16418worker2 # [ 175.450533] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:52Z level=debug event=want client=5 drv=cnf7j5vjqyj799isbg26wgjnrlk208gi-farm-b.drv cached=06419worker1 # [ 175.405076] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:52Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=3223c4sl4ncpldbvw558vqwddijwzfpd-farm-a.drv assign_id=7 outputs=16420worker1 # [ 175.422571] nix-daemon[844]: reaped child process 1711, status = succeeded6421worker1 # [ 175.455242] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:52Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546422worker1 # [ 175.470794] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:52Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546423worker1 # [ 175.485195] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:52Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546424worker1 # [ 175.499074] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6425worker1 # [ 175.704253] nix-daemon[844]: accepted connection from pid 1736, user nix-grpc-daemon (trusted)6426worker1 # [ 175.723003] nix-daemon[844]: reaped child process 1739, status = succeeded6427worker1 # [ 175.894181] nix-daemon[844]: accepted connection from pid 1741, user nix-grpc-daemon (trusted)6428worker1 # [ 175.911449] nix-daemon[844]: reaped child process 1744, status = succeeded6429lb # [ 175.943225] niks3-server[1173]: 2026/09/22 10:38:52 INFO Received uploads request method=POST path=/api/pending_closures6430worker1 # [ 175.936184] nix-grpc-daemon[1688]: time=2026-09-22T10:38:52.567Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6431worker1 # [ 175.938335] nix-grpc-daemon[1688]: time=2026-09-22T10:38:52.568Z level=INFO msg="Uploading pplpzcnjfn295nm16cg0iihpfc9iffry-farm-a.drv (512B)"6432lb # [ 175.974496] niks3-server[1173]: 2026/09/22 10:38:52 INFO Registered completed upload object_key=nar/0g5bxr683biqh358mkd9xrclwllan4cgnv4pzijas586xj68wwx0.nar.zst6433worker1 # [ 175.968883] nix-grpc-daemon[1688]: time=2026-09-22T10:38:52.599Z level=INFO msg="Uploading 1 narinfos"6434lb # [ 175.989508] niks3-server[1173]: 2026/09/22 10:38:52 INFO Received sign narinfos request method=POST path=/api/pending_closures/56/sign6435lb # [ 176.001101] niks3-server[1173]: 2026/09/22 10:38:52 INFO Signed narinfos id=56 count=16436lb # [ 176.005028] niks3-server[1173]: 2026/09/22 10:38:52 INFO Registered completed upload object_key=pplpzcnjfn295nm16cg0iihpfc9iffry.ls6437worker1 # [ 175.996843] nix-grpc-daemon[1688]: time=2026-09-22T10:38:52.627Z level=INFO msg="Upload complete. (481ms)"6438lb # [ 176.014280] niks3-server[1173]: 2026/09/22 10:38:52 INFO Received complete upload request method=POST path=/api/pending_closures/56/complete6439lb # [ 176.019877] niks3-server[1173]: 2026/09/22 10:38:52 INFO Completed upload id=566440lb # [ 176.023617] niks3-server[1173]: 2026/09/22 10:38:52 INFO Registered completed upload object_key=pplpzcnjfn295nm16cg0iihpfc9iffry.narinfo6441worker1 # [ 176.003022] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:52Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=5126442worker1 # [ 176.016544] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:52Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546443worker1 # [ 176.025197] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6444worker1 # [ 176.027911] nix-daemon[844]: reaped child process 1734, status = succeeded6445worker1 # [ 176.038368] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6446worker1 # [ 177.067348] nix-daemon[844]: reaped child process 1748, status = succeeded6447worker1 # [ 177.224892] nix-daemon[844]: accepted connection from pid 1764, user nix-grpc-daemon (trusted)6448worker1 # [ 177.242886] nix-daemon[844]: reaped child process 1767, status = succeeded6449lb # [ 177.274536] niks3-server[1173]: 2026/09/22 10:38:53 INFO Received uploads request method=POST path=/api/pending_closures6450worker1 # [ 177.265263] nix-grpc-daemon[1688]: time=2026-09-22T10:38:53.895Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6451worker1 # [ 177.266746] nix-grpc-daemon[1688]: time=2026-09-22T10:38:53.895Z level=INFO msg="Uploading fhc1zkcygpzppf0bj5fr2k85lf7p87v9-farm-a (128B)"6452lb # [ 177.306159] niks3-server[1173]: 2026/09/22 10:38:53 INFO Registered completed upload object_key=nar/09na6djn66dpys8h08gqw39p0vw7vxqsdzbiw55rknyna8s239bm.nar.zst6453lb # [ 177.320584] niks3-server[1173]: 2026/09/22 10:38:53 INFO Registered completed upload object_key=log/pplpzcnjfn295nm16cg0iihpfc9iffry-farm-a.drv6454lb # [ 177.328610] niks3-server[1173]: 2026/09/22 10:38:53 INFO Received sign narinfos request method=POST path=/api/pending_closures/57/sign6455worker1 # [ 177.318039] nix-grpc-daemon[1688]: time=2026-09-22T10:38:53.948Z level=INFO msg="Uploading 1 narinfos"6456lb # [ 177.334583] niks3-server[1173]: 2026/09/22 10:38:53 INFO Signed narinfos id=57 count=16457lb # [ 177.344224] niks3-server[1173]: 2026/09/22 10:38:53 INFO Registered completed upload object_key=fhc1zkcygpzppf0bj5fr2k85lf7p87v9.ls6458lb # [ 177.352314] niks3-server[1173]: 2026/09/22 10:38:53 INFO Received complete upload request method=POST path=/api/pending_closures/57/complete6459worker1 # [ 177.340046] nix-grpc-daemon[1688]: time=2026-09-22T10:38:53.970Z level=INFO msg="Upload complete. (287ms)"6460lb # [ 177.358397] niks3-server[1173]: 2026/09/22 10:38:53 INFO Completed upload id=576461lb # [ 177.361040] niks3-server[1173]: 2026/09/22 10:38:53 INFO Registered completed upload object_key=fhc1zkcygpzppf0bj5fr2k85lf7p87v9.narinfo6462worker1 # [ 177.348980] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:53Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=pplpzcnjfn295nm16cg0iihpfc9iffry-farm-a.drv assign_id=8 outputs=16463worker2 # [ 177.386338] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:54Z level=debug event=assigned drv=sy43wp9v52pni1wms6q0p3qybpc2pwad-farm-a.drv worker=192.168.1.3:50051 clients=16464worker2 # [ 177.399900] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:54Z level=debug event=want client=5 drv=7vjzkx8444zm2h7882vsj6bmj73zqjq3-farm-b.drv cached=06465worker1 # [ 177.367509] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:53Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546466worker1 # [ 177.371971] nix-daemon[844]: reaped child process 1746, status = succeeded6467worker1 # [ 177.379465] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:54Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546468worker1 # [ 177.390688] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:54Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546469worker1 # [ 177.401047] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6470worker1 # [ 177.588167] nix-daemon[844]: accepted connection from pid 1771, user nix-grpc-daemon (trusted)6471worker1 # [ 177.606842] nix-daemon[844]: reaped child process 1774, status = succeeded6472worker1 # [ 177.781261] nix-daemon[844]: accepted connection from pid 1776, user nix-grpc-daemon (trusted)6473worker1 # [ 177.799425] nix-daemon[844]: reaped child process 1779, status = succeeded6474lb # [ 177.827592] niks3-server[1173]: 2026/09/22 10:38:54 INFO Received uploads request method=POST path=/api/pending_closures6475worker1 # [ 177.819553] nix-grpc-daemon[1688]: time=2026-09-22T10:38:54.450Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6476worker1 # [ 177.821170] nix-grpc-daemon[1688]: time=2026-09-22T10:38:54.450Z level=INFO msg="Uploading sy43wp9v52pni1wms6q0p3qybpc2pwad-farm-a.drv (512B)"6477lb # [ 177.853503] niks3-server[1173]: 2026/09/22 10:38:54 INFO Registered completed upload object_key=nar/1fxz200fmpbckafrn03b2snx9knqpbwf8rwqlbvg68356qj7aj5w.nar.zst6478worker1 # [ 177.850944] nix-grpc-daemon[1688]: time=2026-09-22T10:38:54.481Z level=INFO msg="Uploading 1 narinfos"6479lb # [ 177.870464] niks3-server[1173]: 2026/09/22 10:38:54 INFO Received sign narinfos request method=POST path=/api/pending_closures/58/sign6480lb # [ 177.881459] niks3-server[1173]: 2026/09/22 10:38:54 INFO Signed narinfos id=58 count=16481worker1 # [ 177.874491] nix-grpc-daemon[1688]: time=2026-09-22T10:38:54.505Z level=INFO msg="Upload complete. (461ms)"6482lb # [ 177.886943] niks3-server[1173]: 2026/09/22 10:38:54 INFO Registered completed upload object_key=sy43wp9v52pni1wms6q0p3qybpc2pwad.ls6483lb # [ 177.897054] niks3-server[1173]: 2026/09/22 10:38:54 INFO Received complete upload request method=POST path=/api/pending_closures/58/complete6484lb # [ 177.901807] niks3-server[1173]: 2026/09/22 10:38:54 INFO Completed upload id=586485worker1 # [ 177.884266] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:54Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=5126486lb # [ 177.904239] niks3-server[1173]: 2026/09/22 10:38:54 INFO Registered completed upload object_key=sy43wp9v52pni1wms6q0p3qybpc2pwad.narinfo6487worker1 # [ 177.893041] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:54Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546488worker1 # [ 177.899059] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6489worker1 # [ 177.903310] nix-daemon[844]: reaped child process 1769, status = succeeded6490worker1 # [ 177.913577] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6491worker1 # [ 178.934192] nix-daemon[844]: reaped child process 1783, status = succeeded6492worker1 # [ 179.094236] nix-daemon[844]: accepted connection from pid 1799, user nix-grpc-daemon (trusted)6493worker1 # [ 179.111903] nix-daemon[844]: reaped child process 1802, status = succeeded6494lb # [ 179.146397] niks3-server[1173]: 2026/09/22 10:38:55 INFO Received uploads request method=POST path=/api/pending_closures6495worker1 # [ 179.138103] nix-grpc-daemon[1688]: time=2026-09-22T10:38:55.767Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6496worker1 # [ 179.139551] nix-grpc-daemon[1688]: time=2026-09-22T10:38:55.767Z level=INFO msg="Uploading g5laypvfk2469p8binw3mksk2hv7zhnh-farm-a (128B)"6497lb # [ 179.182307] niks3-server[1173]: 2026/09/22 10:38:55 INFO Registered completed upload object_key=nar/1j2671k9mycq8lxyfx5k6a66d5narfqw2wqwlpwxgwzi93vnvbdn.nar.zst6498worker1 # [ 179.177703] nix-grpc-daemon[1688]: time=2026-09-22T10:38:55.808Z level=INFO msg="Uploading 1 narinfos"6499lb # [ 179.199273] niks3-server[1173]: 2026/09/22 10:38:55 INFO Registered completed upload object_key=log/sy43wp9v52pni1wms6q0p3qybpc2pwad-farm-a.drv6500lb # [ 179.208299] niks3-server[1173]: 2026/09/22 10:38:55 INFO Received sign narinfos request method=POST path=/api/pending_closures/59/sign6501lb # [ 179.212457] niks3-server[1173]: 2026/09/22 10:38:55 INFO Signed narinfos id=59 count=16502lb # [ 179.218377] niks3-server[1173]: 2026/09/22 10:38:55 INFO Registered completed upload object_key=g5laypvfk2469p8binw3mksk2hv7zhnh.ls6503lb # [ 179.221859] niks3-server[1173]: 2026/09/22 10:38:55 INFO Received complete upload request method=POST path=/api/pending_closures/59/complete6504lb # [ 179.225997] niks3-server[1173]: 2026/09/22 10:38:55 INFO Completed upload id=596505worker1 # [ 179.208744] nix-grpc-daemon[1688]: time=2026-09-22T10:38:55.839Z level=INFO msg="Upload complete. (284ms)"6506lb # [ 179.228844] niks3-server[1173]: 2026/09/22 10:38:55 INFO Registered completed upload object_key=g5laypvfk2469p8binw3mksk2hv7zhnh.narinfo6507worker1 # [ 179.216752] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:55Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=sy43wp9v52pni1wms6q0p3qybpc2pwad-farm-a.drv assign_id=9 outputs=16508worker2 # [ 179.253360] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:55Z level=debug event=assigned drv=cnf7j5vjqyj799isbg26wgjnrlk208gi-farm-b.drv worker=192.168.1.3:50051 clients=16509worker2 # [ 179.266505] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:55Z level=debug event=want client=5 drv=cgwyij19i3fl7sqlyz54vzfr4zajg077-farm-b.drv cached=06510worker1 # [ 179.234189] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:55Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546511worker1 # [ 179.239190] nix-daemon[844]: reaped child process 1781, status = succeeded6512worker1 # [ 179.246273] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:55Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546513worker1 # [ 179.258091] nix-grpc-daemon[1688]: time=2026-09-22T10:38:55.889Z level=INFO msg="All 1 paths already cached"6514worker1 # [ 179.265187] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:55Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546515worker1 # [ 179.275486] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6516worker1 # [ 179.459431] nix-daemon[844]: accepted connection from pid 1806, user nix-grpc-daemon (trusted)6517worker1 # [ 179.479945] nix-daemon[844]: reaped child process 1809, status = succeeded6518worker1 # [ 179.656900] nix-daemon[844]: accepted connection from pid 1811, user nix-grpc-daemon (trusted)6519worker1 # [ 179.674767] nix-daemon[844]: reaped child process 1814, status = succeeded6520lb # [ 179.702751] niks3-server[1173]: 2026/09/22 10:38:56 INFO Received uploads request method=POST path=/api/pending_closures6521worker1 # [ 179.694856] nix-grpc-daemon[1688]: time=2026-09-22T10:38:56.325Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6522worker1 # [ 179.696424] nix-grpc-daemon[1688]: time=2026-09-22T10:38:56.325Z level=INFO msg="Uploading cnf7j5vjqyj799isbg26wgjnrlk208gi-farm-b.drv (624B)"6523lb # [ 179.732668] niks3-server[1173]: 2026/09/22 10:38:56 INFO Registered completed upload object_key=nar/1w4sykcjji9zlw0a5xibxycy6x3jyb4drd29x2k9g7x5whfwwpsq.nar.zst6524lb # [ 179.741260] niks3-server[1173]: 2026/09/22 10:38:56 INFO Received sign narinfos request method=POST path=/api/pending_closures/60/sign6525worker1 # [ 179.730320] nix-grpc-daemon[1688]: time=2026-09-22T10:38:56.361Z level=INFO msg="Uploading 1 narinfos"6526lb # [ 179.754187] niks3-server[1173]: 2026/09/22 10:38:56 INFO Signed narinfos id=60 count=16527lb # [ 179.760373] niks3-server[1173]: 2026/09/22 10:38:56 INFO Registered completed upload object_key=cnf7j5vjqyj799isbg26wgjnrlk208gi.ls6528lb # [ 179.767537] niks3-server[1173]: 2026/09/22 10:38:56 INFO Received complete upload request method=POST path=/api/pending_closures/60/complete6529lb # [ 179.773557] niks3-server[1173]: 2026/09/22 10:38:56 INFO Registered completed upload object_key=cnf7j5vjqyj799isbg26wgjnrlk208gi.narinfo6530lb # [ 179.778610] niks3-server[1173]: 2026/09/22 10:38:56 INFO Completed upload id=606531worker1 # [ 179.761824] nix-grpc-daemon[1688]: time=2026-09-22T10:38:56.392Z level=INFO msg="Upload complete. (474ms)"6532worker1 # [ 179.768306] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:56Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=6246533worker1 # [ 179.782401] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:56Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546534worker1 # [ 179.789637] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6535worker1 # [ 179.794526] nix-daemon[844]: reaped child process 1804, status = succeeded6536lb # [ 179.809523] envoy[896]: conn peer=[2001:db8:1::3]:50116 tls=TLSv1.3 subject="CN=worker-1" sni=- flags=- tls_fail="-" rx=986 tx=425 ms=271926537worker1 # [ 179.807771] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6538worker1 # [ 180.631518] nix-daemon[844]: reaped child process 1818, status = succeeded6539worker1 # [ 180.801297] nix-daemon[844]: accepted connection from pid 1834, user nix-grpc-daemon (trusted)6540worker1 # [ 180.819886] nix-daemon[844]: reaped child process 1837, status = succeeded6541lb # [ 180.850073] niks3-server[1173]: 2026/09/22 10:38:57 INFO Received uploads request method=POST path=/api/pending_closures6542worker1 # [ 180.844424] nix-grpc-daemon[1688]: time=2026-09-22T10:38:57.474Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6543worker1 # [ 180.846015] nix-grpc-daemon[1688]: time=2026-09-22T10:38:57.474Z level=INFO msg="Uploading p1334cl91hzsgv8bmiqzkwvhbwq24vn1-farm-b (176B)"6544lb # [ 180.899180] niks3-server[1173]: 2026/09/22 10:38:57 INFO Registered completed upload object_key=nar/0pa7j07n0n5p9wcfihhx3357xrxs4h8s57kncplymh51lvzvz2kh.nar.zst6545worker1 # [ 180.891410] nix-grpc-daemon[1688]: time=2026-09-22T10:38:57.522Z level=INFO msg="Uploading 1 narinfos"6546lb # [ 180.912063] niks3-server[1173]: 2026/09/22 10:38:57 INFO Registered completed upload object_key=log/cnf7j5vjqyj799isbg26wgjnrlk208gi-farm-b.drv6547lb # [ 180.915913] niks3-server[1173]: 2026/09/22 10:38:57 INFO Received sign narinfos request method=POST path=/api/pending_closures/61/sign6548lb # [ 180.923991] niks3-server[1173]: 2026/09/22 10:38:57 INFO Signed narinfos id=61 count=16549lb # [ 180.929341] niks3-server[1173]: 2026/09/22 10:38:57 INFO Registered completed upload object_key=p1334cl91hzsgv8bmiqzkwvhbwq24vn1.ls6550worker1 # [ 180.916380] nix-grpc-daemon[1688]: time=2026-09-22T10:38:57.547Z level=INFO msg="Upload complete. (299ms)"6551lb # [ 180.933799] niks3-server[1173]: 2026/09/22 10:38:57 INFO Received complete upload request method=POST path=/api/pending_closures/61/complete6552lb # [ 180.937116] niks3-server[1173]: 2026/09/22 10:38:57 INFO Completed upload id=616553lb # [ 180.940876] niks3-server[1173]: 2026/09/22 10:38:57 INFO Registered completed upload object_key=p1334cl91hzsgv8bmiqzkwvhbwq24vn1.narinfo6554worker2 # [ 180.960906] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:57Z level=debug event=assigned drv=7vjzkx8444zm2h7882vsj6bmj73zqjq3-farm-b.drv worker=192.168.1.3:50051 clients=16555worker1 # [ 180.924865] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:57Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=cnf7j5vjqyj799isbg26wgjnrlk208gi-farm-b.drv assign_id=10 outputs=16556worker1 # [ 180.940732] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:57Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546557worker2 # [ 180.973768] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:57Z level=debug event=want client=5 drv=dca2k4dvnk52m7gijqfq8q99ih3ni8ny-farm-top.drv cached=06558worker1 # [ 180.945399] nix-daemon[844]: reaped child process 1816, status = succeeded6559worker1 # [ 180.952736] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:57Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546560worker1 # [ 180.963743] nix-grpc-daemon[1688]: time=2026-09-22T10:38:57.594Z level=INFO msg="All 1 paths already cached"6561worker1 # [ 180.971052] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:57Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546562worker1 # [ 180.983760] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6563worker1 # [ 181.176184] nix-daemon[844]: accepted connection from pid 1841, user nix-grpc-daemon (trusted)6564worker1 # [ 181.194912] nix-daemon[844]: reaped child process 1844, status = succeeded6565worker1 # [ 181.366390] nix-daemon[844]: accepted connection from pid 1846, user nix-grpc-daemon (trusted)6566worker1 # [ 181.382531] nix-daemon[844]: reaped child process 1849, status = succeeded6567lb # [ 181.410495] niks3-server[1173]: 2026/09/22 10:38:58 INFO Received uploads request method=POST path=/api/pending_closures6568worker1 # [ 181.402540] nix-grpc-daemon[1688]: time=2026-09-22T10:38:58.033Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6569worker1 # [ 181.404172] nix-grpc-daemon[1688]: time=2026-09-22T10:38:58.033Z level=INFO msg="Uploading 7vjzkx8444zm2h7882vsj6bmj73zqjq3-farm-b.drv (624B)"6570lb # [ 181.438930] niks3-server[1173]: 2026/09/22 10:38:58 INFO Registered completed upload object_key=nar/0l0zss9ls4bhz18slrrmz7l1j93xz7424q7314p64rpgrpqf6b7k.nar.zst6571worker1 # [ 181.431766] nix-grpc-daemon[1688]: time=2026-09-22T10:38:58.062Z level=INFO msg="Uploading 1 narinfos"6572lb # [ 181.451779] niks3-server[1173]: 2026/09/22 10:38:58 INFO Received sign narinfos request method=POST path=/api/pending_closures/62/sign6573lb # [ 181.462352] niks3-server[1173]: 2026/09/22 10:38:58 INFO Signed narinfos id=62 count=16574lb # [ 181.469487] niks3-server[1173]: 2026/09/22 10:38:58 INFO Registered completed upload object_key=7vjzkx8444zm2h7882vsj6bmj73zqjq3.ls6575lb # [ 181.477072] niks3-server[1173]: 2026/09/22 10:38:58 INFO Received complete upload request method=POST path=/api/pending_closures/62/complete6576lb # [ 181.482069] niks3-server[1173]: 2026/09/22 10:38:58 INFO Registered completed upload object_key=7vjzkx8444zm2h7882vsj6bmj73zqjq3.narinfo6577lb # [ 181.486544] niks3-server[1173]: 2026/09/22 10:38:58 INFO Completed upload id=626578worker1 # [ 181.469343] nix-grpc-daemon[1688]: time=2026-09-22T10:38:58.100Z level=INFO msg="Upload complete. (473ms)"6579worker1 # [ 181.477535] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:58Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=6246580worker1 # [ 181.490607] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:58Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546581worker1 # [ 181.496419] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6582worker1 # [ 181.500779] nix-daemon[844]: reaped child process 1839, status = succeeded6583worker1 # [ 181.513332] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6584worker1 # [ 182.960767] nix-daemon[844]: reaped child process 1853, status = succeeded6585worker1 # [ 183.143461] nix-daemon[844]: accepted connection from pid 1869, user nix-grpc-daemon (trusted)6586worker1 # [ 183.159265] nix-daemon[844]: reaped child process 1872, status = succeeded6587lb # [ 183.193901] niks3-server[1173]: 2026/09/22 10:38:59 INFO Received uploads request method=POST path=/api/pending_closures6588worker1 # [ 183.187554] nix-grpc-daemon[1688]: time=2026-09-22T10:38:59.815Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6589worker1 # [ 183.189203] nix-grpc-daemon[1688]: time=2026-09-22T10:38:59.815Z level=INFO msg="Uploading nrnwwsxw1z9dz81vri9gl1dkhkrc6vh8-farm-b (176B)"6590lb # [ 183.225838] niks3-server[1173]: 2026/09/22 10:38:59 INFO Registered completed upload object_key=nar/0kqa5m59sp6aj1ixqj6q54wwqxkcn98fw1rc9hfkcdk460wjdl0l.nar.zst6591lb # [ 183.240261] niks3-server[1173]: 2026/09/22 10:38:59 INFO Registered completed upload object_key=log/7vjzkx8444zm2h7882vsj6bmj73zqjq3-farm-b.drv6592lb # [ 183.248709] niks3-server[1173]: 2026/09/22 10:38:59 INFO Received sign narinfos request method=POST path=/api/pending_closures/63/sign6593worker1 # [ 183.236887] nix-grpc-daemon[1688]: time=2026-09-22T10:38:59.867Z level=INFO msg="Uploading 1 narinfos"6594lb # [ 183.257035] niks3-server[1173]: 2026/09/22 10:38:59 INFO Signed narinfos id=63 count=16595lb # [ 183.266408] niks3-server[1173]: 2026/09/22 10:38:59 INFO Received complete upload request method=POST path=/api/pending_closures/63/complete6596worker1 # [ 183.256625] nix-grpc-daemon[1688]: time=2026-09-22T10:38:59.887Z level=INFO msg="Upload complete. (332ms)"6597lb # [ 183.274077] niks3-server[1173]: 2026/09/22 10:38:59 INFO Registered completed upload object_key=nrnwwsxw1z9dz81vri9gl1dkhkrc6vh8.ls6598lb # [ 183.277719] niks3-server[1173]: 2026/09/22 10:38:59 INFO Completed upload id=636599lb # [ 183.280519] niks3-server[1173]: 2026/09/22 10:38:59 INFO Registered completed upload object_key=nrnwwsxw1z9dz81vri9gl1dkhkrc6vh8.narinfo6600worker2 # [ 183.300704] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:59Z level=debug event=assigned drv=cgwyij19i3fl7sqlyz54vzfr4zajg077-farm-b.drv worker=192.168.1.3:50051 clients=16601worker1 # [ 183.268837] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:59Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=7vjzkx8444zm2h7882vsj6bmj73zqjq3-farm-b.drv assign_id=11 outputs=16602worker2 # [ 183.314537] nix-grpc-daemon[1680]: ts=2026-09-22T10:38:59Z level=debug event=want client=5 drv=2bdcl1cxw40gyh0sjgz98vb0jixgnrd9-farm-top.drv cached=06603worker1 # [ 183.284811] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:59Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546604worker1 # [ 183.289961] nix-daemon[844]: reaped child process 1851, status = succeeded6605worker1 # [ 183.299582] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:59Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546606worker1 # [ 183.312895] nix-grpc-daemon[1688]: time=2026-09-22T10:38:59.943Z level=INFO msg="All 1 paths already cached"6607worker1 # [ 183.321895] nix-grpc-daemon[1628]: ts=2026-09-22T10:38:59Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546608worker1 # [ 183.334422] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6609worker1 # [ 183.526260] nix-daemon[844]: accepted connection from pid 1876, user nix-grpc-daemon (trusted)6610worker1 # [ 183.545286] nix-daemon[844]: reaped child process 1879, status = succeeded6611worker1 # [ 183.719302] nix-daemon[844]: accepted connection from pid 1881, user nix-grpc-daemon (trusted)6612worker1 # [ 183.735124] nix-daemon[844]: reaped child process 1884, status = succeeded6613lb # [ 183.767534] niks3-server[1173]: 2026/09/22 10:39:00 INFO Received uploads request method=POST path=/api/pending_closures6614worker1 # [ 183.763014] nix-grpc-daemon[1688]: time=2026-09-22T10:39:00.393Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6615worker1 # [ 183.764589] nix-grpc-daemon[1688]: time=2026-09-22T10:39:00.393Z level=INFO msg="Uploading cgwyij19i3fl7sqlyz54vzfr4zajg077-farm-b.drv (624B)"6616lb # [ 183.800587] niks3-server[1173]: 2026/09/22 10:39:00 INFO Registered completed upload object_key=nar/0f6g0vggy0qbqsj64l1nivyqdh6q12kh8620ysnf210ani0727vf.nar.zst6617lb # [ 183.812215] niks3-server[1173]: 2026/09/22 10:39:00 INFO Received sign narinfos request method=POST path=/api/pending_closures/64/sign6618worker1 # [ 183.802594] nix-grpc-daemon[1688]: time=2026-09-22T10:39:00.433Z level=INFO msg="Uploading 1 narinfos"6619lb # [ 183.819572] niks3-server[1173]: 2026/09/22 10:39:00 INFO Signed narinfos id=64 count=16620lb # [ 183.826594] niks3-server[1173]: 2026/09/22 10:39:00 INFO Registered completed upload object_key=cgwyij19i3fl7sqlyz54vzfr4zajg077.ls6621lb # [ 183.836087] niks3-server[1173]: 2026/09/22 10:39:00 INFO Received complete upload request method=POST path=/api/pending_closures/64/complete6622lb # [ 183.841535] niks3-server[1173]: 2026/09/22 10:39:00 INFO Completed upload id=646623worker1 # [ 183.824408] nix-grpc-daemon[1688]: time=2026-09-22T10:39:00.455Z level=INFO msg="Upload complete. (475ms)"6624lb # [ 183.844331] niks3-server[1173]: 2026/09/22 10:39:00 INFO Registered completed upload object_key=cgwyij19i3fl7sqlyz54vzfr4zajg077.narinfo6625worker1 # [ 183.832127] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:00Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=6246626worker1 # [ 183.840792] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:00Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546627worker1 # [ 183.851990] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6628worker1 # [ 183.856919] nix-daemon[844]: reaped child process 1874, status = succeeded6629worker1 # [ 183.869809] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6630worker1 # [ 185.119015] nix-daemon[844]: reaped child process 1888, status = succeeded6631worker1 # [ 185.291163] nix-daemon[844]: accepted connection from pid 1904, user nix-grpc-daemon (trusted)6632worker1 # [ 185.308313] nix-daemon[844]: reaped child process 1907, status = succeeded6633lb # [ 185.338667] niks3-server[1173]: 2026/09/22 10:39:01 INFO Received uploads request method=POST path=/api/pending_closures6634worker1 # [ 185.332636] nix-grpc-daemon[1688]: time=2026-09-22T10:39:01.962Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6635worker1 # [ 185.335057] nix-grpc-daemon[1688]: time=2026-09-22T10:39:01.962Z level=INFO msg="Uploading hfsg1k13f2w2fjgkzxk1qi2fh35fdlk5-farm-b (176B)"6636lb # [ 185.388184] niks3-server[1173]: 2026/09/22 10:39:02 INFO Registered completed upload object_key=nar/0x964sxcxsdh3nfjarc1ib52kr50kl8zxl7h34jq5ydx763drras.nar.zst6637worker1 # [ 185.380881] nix-grpc-daemon[1688]: time=2026-09-22T10:39:02.011Z level=INFO msg="Uploading 1 narinfos"6638lb # [ 185.399565] niks3-server[1173]: 2026/09/22 10:39:02 INFO Received sign narinfos request method=POST path=/api/pending_closures/65/sign6639lb # [ 185.409525] niks3-server[1173]: 2026/09/22 10:39:02 INFO Signed narinfos id=65 count=16640lb # [ 185.414518] niks3-server[1173]: 2026/09/22 10:39:02 INFO Registered completed upload object_key=log/cgwyij19i3fl7sqlyz54vzfr4zajg077-farm-b.drv6641lb # [ 185.425304] niks3-server[1173]: 2026/09/22 10:39:02 INFO Registered completed upload object_key=hfsg1k13f2w2fjgkzxk1qi2fh35fdlk5.ls6642lb # [ 185.429962] niks3-server[1173]: 2026/09/22 10:39:02 INFO Received complete upload request method=POST path=/api/pending_closures/65/complete6643lb # [ 185.435054] niks3-server[1173]: 2026/09/22 10:39:02 INFO Completed upload id=656644worker1 # [ 185.417740] nix-grpc-daemon[1688]: time=2026-09-22T10:39:02.048Z level=INFO msg="Upload complete. (336ms)"6645lb # [ 185.438175] niks3-server[1173]: 2026/09/22 10:39:02 INFO Registered completed upload object_key=hfsg1k13f2w2fjgkzxk1qi2fh35fdlk5.narinfo6646worker2 # [ 185.463353] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:02Z level=debug event=assigned drv=dca2k4dvnk52m7gijqfq8q99ih3ni8ny-farm-top.drv worker=192.168.1.3:50051 clients=16647worker1 # [ 185.431807] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:02Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=cgwyij19i3fl7sqlyz54vzfr4zajg077-farm-b.drv assign_id=12 outputs=16648worker2 # [ 185.475927] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:02Z level=debug event=want client=5 drv=qx10pa00pilyp5xdzd48l88d5jz143mr-farm-top.drv cached=06649worker1 # [ 185.452289] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:02Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546650worker1 # [ 185.458450] nix-daemon[844]: reaped child process 1886, status = succeeded6651worker1 # [ 185.466359] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:02Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546652worker1 # [ 185.478726] nix-grpc-daemon[1688]: time=2026-09-22T10:39:02.109Z level=INFO msg="All 2 paths already cached"6653worker1 # [ 185.488760] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:02Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546654worker1 # [ 185.501248] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6655worker1 # [ 185.699821] nix-daemon[844]: accepted connection from pid 1911, user nix-grpc-daemon (trusted)6656worker1 # [ 185.715742] nix-daemon[844]: reaped child process 1914, status = succeeded6657worker1 # [ 185.893520] nix-daemon[844]: accepted connection from pid 1916, user nix-grpc-daemon (trusted)6658worker1 # [ 185.909776] nix-daemon[844]: reaped child process 1919, status = succeeded6659lb # [ 185.937624] niks3-server[1173]: 2026/09/22 10:39:02 INFO Received uploads request method=POST path=/api/pending_closures6660worker1 # [ 185.929936] nix-grpc-daemon[1688]: time=2026-09-22T10:39:02.560Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6661worker1 # [ 185.931406] nix-grpc-daemon[1688]: time=2026-09-22T10:39:02.560Z level=INFO msg="Uploading dca2k4dvnk52m7gijqfq8q99ih3ni8ny-farm-top.drv (752B)"6662lb # [ 185.966506] niks3-server[1173]: 2026/09/22 10:39:02 INFO Registered completed upload object_key=nar/0s7ff7rwqx5p9mdzddqgzd9bbd5j8njfc62k07mgnpy19810bs86.nar.zst6663worker1 # [ 185.959551] nix-grpc-daemon[1688]: time=2026-09-22T10:39:02.590Z level=INFO msg="Uploading 1 narinfos"6664lb # [ 185.982726] niks3-server[1173]: 2026/09/22 10:39:02 INFO Received sign narinfos request method=POST path=/api/pending_closures/66/sign6665lb # [ 185.992565] niks3-server[1173]: 2026/09/22 10:39:02 INFO Signed narinfos id=66 count=16666worker1 # [ 185.982821] nix-grpc-daemon[1688]: time=2026-09-22T10:39:02.613Z level=INFO msg="Upload complete. (466ms)"6667lb # [ 186.001630] niks3-server[1173]: 2026/09/22 10:39:02 INFO Registered completed upload object_key=dca2k4dvnk52m7gijqfq8q99ih3ni8ny.ls6668worker1 # [ 185.990681] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:02Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=7526669lb # [ 186.006676] niks3-server[1173]: 2026/09/22 10:39:02 INFO Received complete upload request method=POST path=/api/pending_closures/66/complete6670lb # [ 186.012841] niks3-server[1173]: 2026/09/22 10:39:02 INFO Completed upload id=666671worker1 # [ 185.996294] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:02Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546672lb # [ 186.015410] niks3-server[1173]: 2026/09/22 10:39:02 INFO Registered completed upload object_key=dca2k4dvnk52m7gijqfq8q99ih3ni8ny.narinfo6673worker1 # [ 186.005976] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6674worker1 # [ 186.010097] nix-daemon[844]: reaped child process 1909, status = succeeded6675worker1 # [ 186.019918] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6676worker1 # [ 186.855329] nix-daemon[844]: reaped child process 1923, status = succeeded6677worker1 # [ 187.020894] nix-daemon[844]: accepted connection from pid 1939, user nix-grpc-daemon (trusted)6678worker1 # [ 187.037592] nix-daemon[844]: reaped child process 1942, status = succeeded6679lb # [ 187.069543] niks3-server[1173]: 2026/09/22 10:39:03 INFO Received uploads request method=POST path=/api/pending_closures6680worker1 # [ 187.064394] nix-grpc-daemon[1688]: time=2026-09-22T10:39:03.692Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6681worker1 # [ 187.065883] nix-grpc-daemon[1688]: time=2026-09-22T10:39:03.693Z level=INFO msg="Uploading d88xgckxx0s0n5ly2d0jw1qy1641z8pq-farm-top (232B)"6682lb # [ 187.107084] niks3-server[1173]: 2026/09/22 10:39:03 INFO Registered completed upload object_key=log/dca2k4dvnk52m7gijqfq8q99ih3ni8ny-farm-top.drv6683lb # [ 187.119854] niks3-server[1173]: 2026/09/22 10:39:03 INFO Registered completed upload object_key=nar/0knxhkhz3x04vk8as5wv5ar2d0h02prv7q5f42qm1xjgr70ancd0.nar.zst6684lb # [ 187.132196] niks3-server[1173]: 2026/09/22 10:39:03 INFO Received sign narinfos request method=POST path=/api/pending_closures/67/sign6685worker1 # [ 187.121851] nix-grpc-daemon[1688]: time=2026-09-22T10:39:03.752Z level=INFO msg="Uploading 1 narinfos"6686lb # [ 187.142452] niks3-server[1173]: 2026/09/22 10:39:03 INFO Signed narinfos id=67 count=16687lb # [ 187.148242] niks3-server[1173]: 2026/09/22 10:39:03 INFO Registered completed upload object_key=d88xgckxx0s0n5ly2d0jw1qy1641z8pq.ls6688lb # [ 187.155616] niks3-server[1173]: 2026/09/22 10:39:03 INFO Received complete upload request method=POST path=/api/pending_closures/67/complete6689lb # [ 187.163334] niks3-server[1173]: 2026/09/22 10:39:03 INFO Completed upload id=676690worker1 # [ 187.146508] nix-grpc-daemon[1688]: time=2026-09-22T10:39:03.777Z level=INFO msg="Upload complete. (316ms)"6691lb # [ 187.166629] niks3-server[1173]: 2026/09/22 10:39:03 INFO Registered completed upload object_key=d88xgckxx0s0n5ly2d0jw1qy1641z8pq.narinfo6692worker2 # [ 187.191543] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:03Z level=debug event=assigned drv=2bdcl1cxw40gyh0sjgz98vb0jixgnrd9-farm-top.drv worker=192.168.1.3:50051 clients=16693worker1 # [ 187.157886] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:03Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=dca2k4dvnk52m7gijqfq8q99ih3ni8ny-farm-top.drv assign_id=13 outputs=16694worker1 # [ 187.174669] nix-daemon[844]: reaped child process 1921, status = succeeded6695worker1 # [ 187.203698] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:03Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546696worker1 # [ 187.218276] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:03Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546697worker1 # [ 187.232355] nix-grpc-daemon[1688]: time=2026-09-22T10:39:03.863Z level=INFO msg="All 2 paths already cached"6698worker1 # [ 187.242273] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:03Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546699worker1 # [ 187.255944] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6700worker1 # [ 187.452250] nix-daemon[844]: accepted connection from pid 1946, user nix-grpc-daemon (trusted)6701worker1 # [ 187.472816] nix-daemon[844]: reaped child process 1949, status = succeeded6702worker1 # [ 187.649062] nix-daemon[844]: accepted connection from pid 1951, user nix-grpc-daemon (trusted)6703worker1 # [ 187.669160] nix-daemon[844]: reaped child process 1954, status = succeeded6704lb # [ 187.697341] niks3-server[1173]: 2026/09/22 10:39:04 INFO Received uploads request method=POST path=/api/pending_closures6705worker1 # [ 187.689340] nix-grpc-daemon[1688]: time=2026-09-22T10:39:04.320Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6706worker1 # [ 187.690830] nix-grpc-daemon[1688]: time=2026-09-22T10:39:04.320Z level=INFO msg="Uploading 2bdcl1cxw40gyh0sjgz98vb0jixgnrd9-farm-top.drv (752B)"6707lb # [ 187.726945] niks3-server[1173]: 2026/09/22 10:39:04 INFO Registered completed upload object_key=nar/0cpiyiv7frlhngqhhc703bglz5q2cpg4n8sgds4sfg7nvw0fcdn5.nar.zst6708worker1 # [ 187.721394] nix-grpc-daemon[1688]: time=2026-09-22T10:39:04.352Z level=INFO msg="Uploading 1 narinfos"6709lb # [ 187.741263] niks3-server[1173]: 2026/09/22 10:39:04 INFO Received sign narinfos request method=POST path=/api/pending_closures/68/sign6710lb # [ 187.751277] niks3-server[1173]: 2026/09/22 10:39:04 INFO Signed narinfos id=68 count=16711lb # [ 187.755317] niks3-server[1173]: 2026/09/22 10:39:04 INFO Registered completed upload object_key=2bdcl1cxw40gyh0sjgz98vb0jixgnrd9.ls6712worker1 # [ 187.745648] nix-grpc-daemon[1688]: time=2026-09-22T10:39:04.376Z level=INFO msg="Upload complete. (474ms)"6713lb # [ 187.763905] niks3-server[1173]: 2026/09/22 10:39:04 INFO Received complete upload request method=POST path=/api/pending_closures/68/complete6714lb # [ 187.768858] niks3-server[1173]: 2026/09/22 10:39:04 INFO Completed upload id=686715worker1 # [ 187.752131] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:04Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=7526716lb # [ 187.772699] niks3-server[1173]: 2026/09/22 10:39:04 INFO Registered completed upload object_key=2bdcl1cxw40gyh0sjgz98vb0jixgnrd9.narinfo6717worker1 # [ 187.760589] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:04Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546718worker1 # [ 187.770807] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6719worker1 # [ 187.775045] nix-daemon[844]: reaped child process 1944, status = succeeded6720worker1 # [ 187.787388] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6721worker1 # [ 188.618454] nix-daemon[844]: reaped child process 1958, status = succeeded6722worker1 # [ 188.780580] nix-daemon[844]: accepted connection from pid 1974, user nix-grpc-daemon (trusted)6723worker1 # [ 188.799294] nix-daemon[844]: reaped child process 1977, status = succeeded6724lb # [ 188.833439] niks3-server[1173]: 2026/09/22 10:39:05 INFO Received uploads request method=POST path=/api/pending_closures6725worker1 # [ 188.827525] nix-grpc-daemon[1688]: time=2026-09-22T10:39:05.456Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6726worker1 # [ 188.829212] nix-grpc-daemon[1688]: time=2026-09-22T10:39:05.456Z level=INFO msg="Uploading vz84kyfcw804d0iyx7nj61j47ms9zxmn-farm-top (232B)"6727lb # [ 188.868341] niks3-server[1173]: 2026/09/22 10:39:05 INFO Registered completed upload object_key=log/2bdcl1cxw40gyh0sjgz98vb0jixgnrd9-farm-top.drv6728lb # [ 188.881724] niks3-server[1173]: 2026/09/22 10:39:05 INFO Registered completed upload object_key=nar/0aaz4iim7fa45zws98gbhp73l23kbm0fvp4anssnxa96v666ljhs.nar.zst6729lb # [ 188.893306] niks3-server[1173]: 2026/09/22 10:39:05 INFO Received sign narinfos request method=POST path=/api/pending_closures/69/sign6730worker1 # [ 188.882303] nix-grpc-daemon[1688]: time=2026-09-22T10:39:05.512Z level=INFO msg="Uploading 1 narinfos"6731lb # [ 188.900752] niks3-server[1173]: 2026/09/22 10:39:05 INFO Signed narinfos id=69 count=16732lb # [ 188.905961] niks3-server[1173]: 2026/09/22 10:39:05 INFO Registered completed upload object_key=vz84kyfcw804d0iyx7nj61j47ms9zxmn.ls6733lb # [ 188.915948] niks3-server[1173]: 2026/09/22 10:39:05 INFO Received complete upload request method=POST path=/api/pending_closures/69/complete6734lb # [ 188.923113] niks3-server[1173]: 2026/09/22 10:39:05 INFO Completed upload id=696735worker1 # [ 188.905847] nix-grpc-daemon[1688]: time=2026-09-22T10:39:05.536Z level=INFO msg="Upload complete. (309ms)"6736lb # [ 188.926244] niks3-server[1173]: 2026/09/22 10:39:05 INFO Registered completed upload object_key=vz84kyfcw804d0iyx7nj61j47ms9zxmn.narinfo6737worker2 # [ 188.951172] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:05Z level=debug event=assigned drv=qx10pa00pilyp5xdzd48l88d5jz143mr-farm-top.drv worker=192.168.1.3:50051 clients=16738worker1 # [ 188.917181] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:05Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=2bdcl1cxw40gyh0sjgz98vb0jixgnrd9-farm-top.drv assign_id=14 outputs=16739worker1 # [ 188.933129] nix-daemon[844]: reaped child process 1956, status = succeeded6740worker1 # [ 188.963115] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:05Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546741worker1 # [ 188.978445] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:05Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546742worker1 # [ 188.993877] nix-grpc-daemon[1688]: time=2026-09-22T10:39:05.624Z level=INFO msg="All 2 paths already cached"6743worker1 # [ 189.003395] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:05Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:565546744worker1 # [ 189.017464] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6745worker1 # [ 189.215537] nix-daemon[844]: accepted connection from pid 1981, user nix-grpc-daemon (trusted)6746worker1 # [ 189.234280] nix-daemon[844]: reaped child process 1984, status = succeeded6747worker1 # [ 189.405138] nix-daemon[844]: accepted connection from pid 1986, user nix-grpc-daemon (trusted)6748worker1 # [ 189.423462] nix-daemon[844]: reaped child process 1989, status = succeeded6749lb # [ 189.451865] niks3-server[1173]: 2026/09/22 10:39:06 INFO Received uploads request method=POST path=/api/pending_closures6750worker1 # [ 189.443490] nix-grpc-daemon[1688]: time=2026-09-22T10:39:06.074Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6751worker1 # [ 189.445126] nix-grpc-daemon[1688]: time=2026-09-22T10:39:06.074Z level=INFO msg="Uploading qx10pa00pilyp5xdzd48l88d5jz143mr-farm-top.drv (752B)"6752lb # [ 189.479116] niks3-server[1173]: 2026/09/22 10:39:06 INFO Registered completed upload object_key=nar/05zmsnqx4aj55yncx917mz6d3hzrvz2rx2x75313vvy1kq1zv9fv.nar.zst6753worker1 # [ 189.473038] nix-grpc-daemon[1688]: time=2026-09-22T10:39:06.103Z level=INFO msg="Uploading 1 narinfos"6754lb # [ 189.493939] niks3-server[1173]: 2026/09/22 10:39:06 INFO Received sign narinfos request method=POST path=/api/pending_closures/70/sign6755lb # [ 189.506067] niks3-server[1173]: 2026/09/22 10:39:06 INFO Signed narinfos id=70 count=16756lb # [ 189.513002] niks3-server[1173]: 2026/09/22 10:39:06 INFO Received complete upload request method=POST path=/api/pending_closures/70/complete6757worker1 # [ 189.500395] nix-grpc-daemon[1688]: time=2026-09-22T10:39:06.131Z level=INFO msg="Upload complete. (466ms)"6758lb # [ 189.517977] niks3-server[1173]: 2026/09/22 10:39:06 INFO Registered completed upload object_key=qx10pa00pilyp5xdzd48l88d5jz143mr.ls6759lb # [ 189.523080] niks3-server[1173]: 2026/09/22 10:39:06 INFO Completed upload id=706760worker1 # [ 189.506326] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:06Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=0 paths=1 nar_bytes_in=7526761lb # [ 189.526459] niks3-server[1173]: 2026/09/22 10:39:06 INFO Registered completed upload object_key=qx10pa00pilyp5xdzd48l88d5jz143mr.narinfo6762worker1 # [ 189.518899] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:06Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:565546763worker1 # [ 189.524433] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6764worker1 # [ 189.528745] nix-daemon[844]: reaped child process 1979, status = succeeded6765worker1 # [ 189.540384] nix-daemon[844]: accepted connection from pid 1628, user nix-grpc-daemon (trusted)6766worker1 # [ 190.377939] nix-daemon[844]: reaped child process 1993, status = succeeded6767worker1 # [ 190.557752] nix-daemon[844]: accepted connection from pid 2009, user nix-grpc-daemon (trusted)6768worker1 # [ 190.574726] nix-daemon[844]: reaped child process 2012, status = succeeded6769lb # [ 190.610118] niks3-server[1173]: 2026/09/22 10:39:07 INFO Received uploads request method=POST path=/api/pending_closures6770worker1 # [ 190.603590] nix-grpc-daemon[1688]: time=2026-09-22T10:39:07.231Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6771worker1 # [ 190.605484] nix-grpc-daemon[1688]: time=2026-09-22T10:39:07.231Z level=INFO msg="Uploading iagwbb072hnm79qb9wsrjp5907zc7aa2-farm-top (232B)"6772lb # [ 190.646219] niks3-server[1173]: 2026/09/22 10:39:07 INFO Registered completed upload object_key=nar/053jd22j3kv7v6j0q7cspdpc0h1c1xmywmajl6sd5hzwg50z2yj1.nar.zst6773worker1 # [ 190.643256] nix-grpc-daemon[1688]: time=2026-09-22T10:39:07.273Z level=INFO msg="Uploading 1 narinfos"6774lb # [ 190.662157] niks3-server[1173]: 2026/09/22 10:39:07 INFO Received sign narinfos request method=POST path=/api/pending_closures/71/sign6775lb # [ 190.672950] niks3-server[1173]: 2026/09/22 10:39:07 INFO Signed narinfos id=71 count=16776lb # [ 190.675685] niks3-server[1173]: 2026/09/22 10:39:07 INFO Registered completed upload object_key=log/qx10pa00pilyp5xdzd48l88d5jz143mr-farm-top.drv6777lb # [ 190.684960] niks3-server[1173]: 2026/09/22 10:39:07 INFO Registered completed upload object_key=iagwbb072hnm79qb9wsrjp5907zc7aa2.ls6778lb # [ 190.693808] niks3-server[1173]: 2026/09/22 10:39:07 INFO Received complete upload request method=POST path=/api/pending_closures/71/complete6779worker1 # [ 190.672748] nix-grpc-daemon[1688]: time=2026-09-22T10:39:07.303Z level=INFO msg="Upload complete. (323ms)"6780lb # [ 190.699364] niks3-server[1173]: 2026/09/22 10:39:07 INFO Completed upload id=716781lb # [ 190.702780] niks3-server[1173]: 2026/09/22 10:39:07 INFO Registered completed upload object_key=iagwbb072hnm79qb9wsrjp5907zc7aa2.narinfo6782worker1 # [ 190.685134] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:07Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56554 duration_s=1 drv=qx10pa00pilyp5xdzd48l88d5jz143mr-farm-top.drv assign_id=15 outputs=16783worker1 # [ 190.702166] nix-daemon[844]: reaped child process 1991, status = succeeded6784client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto --expr 'map (tag: import /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix { inherit tag; features = ["vip"]; }) ["f1" "f2" "f3"]' --impure 2>&1, in 18.57 seconds)6785worker1: must succeed: curl -sf http://127.0.0.1:9464/metrics6786worker1: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.07 seconds)6787client: must succeed: systemd-run --unit gpu nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&restart-grace=30' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag f5 --arg features '["gpu"]'6788client # [ 190.901547] systemd[1]: Started [systemd-run] /run/current-system/sw/bin/nix build -L --store "grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&restart-grace=30" --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag f5 --arg features "[\"gpu\"]".6789client: (finished: must succeed: systemd-run --unit gpu nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key&restart-grace=30' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag f5 --arg features '["gpu"]', in 0.09 seconds)6790lb: must succeed: curl -sf localhost:9901/clusters6791lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)6792lb: must succeed: curl -sf localhost:9901/clusters6793lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)6794worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics6795worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)6796worker1 # [ 191.164502] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:07Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:565546797client # Running as unit: gpu.service; invocation ID: 5f88311739f04536a108c7454053611e6798client # [ 191.166270] nix[1466]: don't know how to build these paths:6799client # [ 191.168037] nix[1466]: /nix/store/3i1nziwpdqq19lzpqy4rywzxnm8dlps6-farm-top.drv6800worker1 # [ 191.177109] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:07Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546801worker2 # [ 191.237230] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:07Z level=debug event=want client=6 drv=1jn0w6x679ijjxwcfb2gxc6lw2fxim2k-farm-a.drv cached=06802client # [ 191.202903] nix[1466]: warning: 1jn0w6x679ijjxwcfb2gxc6lw2fxim2k-farm-a.drv: no connected, non-draining worker for system 'aarch64-linux' offers features {gpu}, waiting up to 30s for a worker6803lb: must succeed: curl -sf localhost:9901/clusters6804lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)6805lb: must succeed: curl -sf localhost:9901/clusters6806lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)6807worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics6808worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)6809client: waiting for success: systemctl show -p Result gpu | grep -q exit-code6810lb # [ 199.815615] envoy[896]: conn peer=[2001:db8:1::1]:43232 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=27079 tx=12543 ms=182446811client # [ 221.205488] nix[1466]: 1jn0w6x679ijjxwcfb2gxc6lw2fxim2k-farm-a.drv: no connected, non-draining worker for system 'aarch64-linux' offers features {gpu}6812client # [ 221.217857] nix[1466]: error: dependency '1jn0w6x679ijjxwcfb2gxc6lw2fxim2k-farm-a.drv' failed6813client # [ 221.259635] systemd[1]: gpu.service: Main process exited, code=exited, status=1/FAILURE6814client # [ 221.266388] systemd[1]: gpu.service: Failed with result 'exit-code'.6815client # [ 221.270058] systemd[1]: gpu.service: Consumed 179ms CPU time over 30.357s wall clock time, 35.3M memory peak, 5.1K incoming IP traffic, 5K outgoing IP traffic.6816client: (finished: waiting for success: systemctl show -p Result gpu | grep -q exit-code, in 29.25 seconds)6817client: must succeed: journalctl -u gpu -o cat6818client: (finished: must succeed: journalctl -u gpu -o cat, in 0.04 seconds)6819lb: must succeed: curl -sf localhost:9901/clusters6820lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.08 seconds)6821lb: must succeed: curl -sf localhost:9901/clusters6822lb: (finished: must succeed: curl -sf localhost:9901/clusters, in 0.06 seconds)6823worker2: must succeed: curl -sf http://127.0.0.1:9464/metrics6824worker2: (finished: must succeed: curl -sf http://127.0.0.1:9464/metrics, in 0.08 seconds)6825(finished: subtest: requiredSystemFeatures: placed on the worker that has them, refused when none does, in 49.66 seconds)6826subtest: low disk drains a worker and builds go to the other6827worker1: must succeed: fallocate -l $(( $(df --output=avail -B1 /nix/store | tail -1) - 100*1024*1024 )) /nix/.rw-store/fill6828worker1 # [ 222.889259] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:39Z level=info event=unhealthy reason=min_free6829worker1: (finished: must succeed: fallocate -l $(( $(df --output=avail -B1 /nix/store | tail -1) - 100*1024*1024 )) /nix/.rw-store/fill, in 1.13 seconds)6830worker1: waiting for success: journalctl -u nix-grpc-daemon -o cat | grep -q 'event=unhealthy reason=min_free'6831worker1: (finished: waiting for success: journalctl -u nix-grpc-daemon -o cat | grep -q 'event=unhealthy reason=min_free', in 0.04 seconds)6832client: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag drain >&26833client # don't know how to build these paths:6834client # /nix/store/77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv6835worker2 # [ 223.324713] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:39Z level=debug event=rpc_start method=QueryMissing cn=ci-1 peer=ipv4:192.168.1.2:561646836worker1 # [ 223.293602] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:39Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:565546837worker2 # [ 223.355328] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:39Z level=debug event=want client=7 drv=kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv cached=06838worker2 # [ 223.362346] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:39Z level=debug event=assigned drv=kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv worker=192.168.1.4:50051 clients=16839worker2 # [ 223.402382] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:40Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646840worker2 # [ 223.418796] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:40Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646841client # copying 1 paths to 'lb:50051'...6842client # copying path '/nix/store/kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv' to 'lb:50051'...6843worker2 # [ 223.434707] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:40Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:561646844worker2 # [ 223.449621] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6845worker2 # [ 223.647724] nix-daemon[864]: accepted connection from pid 1975, user nix-grpc-daemon (trusted)6846worker2 # [ 223.666865] nix-daemon[864]: reaped child process 1978, status = succeeded6847worker2 # [ 223.838501] nix-daemon[864]: accepted connection from pid 1980, user nix-grpc-daemon (trusted)6848worker2 # [ 223.854079] nix-daemon[864]: reaped child process 1983, status = succeeded6849lb # [ 223.848396] niks3-server[1173]: 2026/09/22 10:39:40 INFO Received uploads request method=POST path=/api/pending_closures6850worker2 # [ 223.878598] nix-grpc-daemon[1760]: time=2026-09-22T10:39:40.522Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6851worker2 # [ 223.880271] nix-grpc-daemon[1760]: time=2026-09-22T10:39:40.522Z level=INFO msg="Uploading kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv (512B)"6852lb # [ 223.882677] niks3-server[1173]: 2026/09/22 10:39:40 INFO Registered completed upload object_key=nar/1sfy0gcq4znfcj1f68c749r5x2jwmwsfbfp6xdflz5xkndjwjjm3.nar.zst6853worker2 # [ 223.923243] nix-grpc-daemon[1760]: time=2026-09-22T10:39:40.567Z level=INFO msg="Uploading 1 narinfos"6854lb # [ 223.896732] niks3-server[1173]: 2026/09/22 10:39:40 INFO Received sign narinfos request method=POST path=/api/pending_closures/72/sign6855lb # [ 223.909086] niks3-server[1173]: 2026/09/22 10:39:40 INFO Signed narinfos id=72 count=16856lb # [ 223.916778] niks3-server[1173]: 2026/09/22 10:39:40 INFO Registered completed upload object_key=kksl1jqfg56vabgqvrkz5vki6nq7r59r.ls6857lb # [ 223.927974] niks3-server[1173]: 2026/09/22 10:39:40 INFO Received complete upload request method=POST path=/api/pending_closures/72/complete6858lb # [ 223.937730] niks3-server[1173]: 2026/09/22 10:39:40 INFO Completed upload id=726859worker2 # [ 223.956066] nix-grpc-daemon[1760]: time=2026-09-22T10:39:40.600Z level=INFO msg="Upload complete. (489ms)"6860lb # [ 223.941305] niks3-server[1173]: 2026/09/22 10:39:40 INFO Registered completed upload object_key=kksl1jqfg56vabgqvrkz5vki6nq7r59r.narinfo6861worker2 # [ 223.966995] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:40Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=0 paths=1 nar_bytes_in=5126862worker2 # [ 223.981355] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:40Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646863worker2 # [ 223.991212] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6864worker2 # [ 223.994645] nix-daemon[864]: reaped child process 1973, status = succeeded6865client # building '/nix/store/kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv' on lb:50051...6866client # farm-a> worker2: building kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv6867worker2 # [ 224.010775] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6868client # farm-a> disabling binary cache 'https://cache.nixos.org' for 60 seconds6869client # farm-a> LOG-farm-a-drain6870worker2 # [ 225.058334] nix-daemon[864]: reaped child process 1987, status = succeeded6871worker2 # [ 225.239130] nix-daemon[864]: accepted connection from pid 2003, user nix-grpc-daemon (trusted)6872worker2 # [ 225.256206] nix-daemon[864]: reaped child process 2006, status = succeeded6873lb # [ 225.250842] niks3-server[1173]: 2026/09/22 10:39:41 INFO Received uploads request method=POST path=/api/pending_closures6874worker2 # [ 225.280409] nix-grpc-daemon[1760]: time=2026-09-22T10:39:41.923Z level=INFO msg="Uploading 1 paths to lb (0 already cached)"6875worker2 # [ 225.282015] nix-grpc-daemon[1760]: time=2026-09-22T10:39:41.923Z level=INFO msg="Uploading 272b39wn7cbnhkcc2rrz2f6kmhhmcwnr-farm-a (128B)"6876lb # [ 225.292744] niks3-server[1173]: 2026/09/22 10:39:41 INFO Registered completed upload object_key=nar/1bf9z7y0ikx07b4zjag42zyrv2pq1ycw4461s579kfpmni9kz4dw.nar.zst6877lb # [ 225.305229] niks3-server[1173]: 2026/09/22 10:39:41 INFO Registered completed upload object_key=log/kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv6878worker2 # [ 225.331599] nix-grpc-daemon[1760]: time=2026-09-22T10:39:41.975Z level=INFO msg="Uploading 1 narinfos"6879lb # [ 225.315734] niks3-server[1173]: 2026/09/22 10:39:41 INFO Received sign narinfos request method=POST path=/api/pending_closures/73/sign6880lb # [ 225.323173] niks3-server[1173]: 2026/09/22 10:39:41 INFO Signed narinfos id=73 count=16881lb # [ 225.327025] niks3-server[1173]: 2026/09/22 10:39:41 INFO Registered completed upload object_key=272b39wn7cbnhkcc2rrz2f6kmhhmcwnr.ls6882lb # [ 225.333891] niks3-server[1173]: 2026/09/22 10:39:41 INFO Received complete upload request method=POST path=/api/pending_closures/73/complete6883lb # [ 225.338423] niks3-server[1173]: 2026/09/22 10:39:41 INFO Completed upload id=736884worker2 # [ 225.355602] nix-grpc-daemon[1760]: time=2026-09-22T10:39:41.999Z level=INFO msg="Upload complete. (332ms)"6885lb # [ 225.340867] niks3-server[1173]: 2026/09/22 10:39:41 INFO Registered completed upload object_key=272b39wn7cbnhkcc2rrz2f6kmhhmcwnr.narinfo6886worker2 # [ 225.370378] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=1 drv=kksl1jqfg56vabgqvrkz5vki6nq7r59r-farm-a.drv assign_id=16 outputs=16887worker2 # [ 225.392321] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=debug event=want client=7 drv=fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv cached=06888worker2 # [ 225.398216] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=debug event=assigned drv=fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv worker=192.168.1.4:50051 clients=16889worker2 # [ 225.403161] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646890worker2 # [ 225.408474] nix-daemon[864]: reaped child process 1985, status = succeeded6891worker2 # [ 225.411577] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646892worker2 # [ 225.423595] nix-grpc-daemon[1760]: time=2026-09-22T10:39:42.067Z level=INFO msg="All 1 paths already cached"6893client # copying 1 paths to 'lb:50051'...6894client # copying path '/nix/store/fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv' to 'lb:50051'...6895worker2 # [ 225.432984] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:561646896worker2 # [ 225.445215] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6897worker2 # [ 225.636967] nix-daemon[864]: accepted connection from pid 2012, user nix-grpc-daemon (trusted)6898worker2 # [ 225.653356] nix-daemon[864]: reaped child process 2015, status = succeeded6899worker2 # [ 225.825675] nix-daemon[864]: accepted connection from pid 2017, user nix-grpc-daemon (trusted)6900worker2 # [ 225.841099] nix-daemon[864]: reaped child process 2020, status = succeeded6901lb # [ 225.834232] niks3-server[1173]: 2026/09/22 10:39:42 INFO Received uploads request method=POST path=/api/pending_closures6902worker2 # [ 225.861007] nix-grpc-daemon[1760]: time=2026-09-22T10:39:42.505Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6903worker2 # [ 225.862462] nix-grpc-daemon[1760]: time=2026-09-22T10:39:42.505Z level=INFO msg="Uploading fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv (632B)"6904lb # [ 225.860173] niks3-server[1173]: 2026/09/22 10:39:42 INFO Registered completed upload object_key=nar/1ax4z88sb8d24gsvydxvgdy2lijkmhnycxhr5war1j3sq97cyzqf.nar.zst6905worker2 # [ 225.889541] nix-grpc-daemon[1760]: time=2026-09-22T10:39:42.533Z level=INFO msg="Uploading 1 narinfos"6906lb # [ 225.875506] niks3-server[1173]: 2026/09/22 10:39:42 INFO Received sign narinfos request method=POST path=/api/pending_closures/74/sign6907lb # [ 225.886385] niks3-server[1173]: 2026/09/22 10:39:42 INFO Signed narinfos id=74 count=16908worker2 # [ 225.914249] nix-grpc-daemon[1760]: time=2026-09-22T10:39:42.558Z level=INFO msg="Upload complete. (455ms)"6909lb # [ 225.896244] niks3-server[1173]: 2026/09/22 10:39:42 INFO Registered completed upload object_key=fa7adcm0x8sbgmrsfx912fp8xakk99g6.ls6910lb # [ 225.900961] niks3-server[1173]: 2026/09/22 10:39:42 INFO Received complete upload request method=POST path=/api/pending_closures/74/complete6911worker2 # [ 225.920195] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=0 paths=1 nar_bytes_in=6326912lb # [ 225.905416] niks3-server[1173]: 2026/09/22 10:39:42 INFO Completed upload id=746913lb # [ 225.907758] niks3-server[1173]: 2026/09/22 10:39:42 INFO Registered completed upload object_key=fa7adcm0x8sbgmrsfx912fp8xakk99g6.narinfo6914worker2 # [ 225.931549] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:42Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646915worker2 # [ 225.937660] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6916worker2 # [ 225.942145] nix-daemon[864]: reaped child process 2010, status = succeeded6917client # building '/nix/store/fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv' on lb:50051...6918worker2 # [ 225.952752] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6919client # farm-b> worker2: building fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv6920client # farm-b> disabling binary cache 'https://cache.nixos.org' for 60 seconds6921client # farm-b> LOG-farm-b-drain6922worker2 # [ 226.781740] nix-daemon[864]: reaped child process 2024, status = succeeded6923worker2 # [ 226.946207] nix-daemon[864]: accepted connection from pid 2040, user nix-grpc-daemon (trusted)6924worker2 # [ 226.962251] nix-daemon[864]: reaped child process 2043, status = succeeded6925lb # [ 226.956908] niks3-server[1173]: 2026/09/22 10:39:43 INFO Received uploads request method=POST path=/api/pending_closures6926worker2 # [ 226.987661] nix-grpc-daemon[1760]: time=2026-09-22T10:39:43.628Z level=INFO msg="Uploading 1 paths to lb (1 already cached)"6927worker2 # [ 226.989320] nix-grpc-daemon[1760]: time=2026-09-22T10:39:43.628Z level=INFO msg="Uploading g774lks418zc45bd9hwhinm61pwcwnr9-farm-b (176B)"6928lb # [ 226.996424] niks3-server[1173]: 2026/09/22 10:39:43 INFO Registered completed upload object_key=nar/1ammzz4zx9sggy8v73zxk3j08vbqxgpw14n8lx2fh57bx24yzvnj.nar.zst6929lb # [ 227.008538] niks3-server[1173]: 2026/09/22 10:39:43 INFO Registered completed upload object_key=log/fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv6930worker2 # [ 227.034380] nix-grpc-daemon[1760]: time=2026-09-22T10:39:43.678Z level=INFO msg="Uploading 1 narinfos"6931lb # [ 227.021056] niks3-server[1173]: 2026/09/22 10:39:43 INFO Received sign narinfos request method=POST path=/api/pending_closures/75/sign6932lb # [ 227.028898] niks3-server[1173]: 2026/09/22 10:39:43 INFO Signed narinfos id=75 count=16933lb # [ 227.035238] niks3-server[1173]: 2026/09/22 10:39:43 INFO Registered completed upload object_key=g774lks418zc45bd9hwhinm61pwcwnr9.ls6934worker2 # [ 227.058050] nix-grpc-daemon[1760]: time=2026-09-22T10:39:43.702Z level=INFO msg="Upload complete. (295ms)"6935lb # [ 227.040596] niks3-server[1173]: 2026/09/22 10:39:43 INFO Received complete upload request method=POST path=/api/pending_closures/75/complete6936lb # [ 227.046449] niks3-server[1173]: 2026/09/22 10:39:43 INFO Completed upload id=756937lb # [ 227.048646] niks3-server[1173]: 2026/09/22 10:39:43 INFO Registered completed upload object_key=g774lks418zc45bd9hwhinm61pwcwnr9.narinfo6938worker2 # [ 227.068954] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:43Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=1 drv=fa7adcm0x8sbgmrsfx912fp8xakk99g6-farm-b.drv assign_id=17 outputs=16939worker2 # [ 227.081683] nix-daemon[864]: reaped child process 2022, status = succeeded6940worker2 # [ 227.116229] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:43Z level=debug event=want client=7 drv=77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv cached=06941worker2 # [ 227.122827] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:43Z level=debug event=assigned drv=77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv worker=192.168.1.4:50051 clients=16942worker2 # [ 227.162155] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:43Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646943worker2 # [ 227.177523] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:43Z level=debug event=rpc_start method=QueryValidPaths cn=ci-1 peer=ipv4:192.168.1.2:561646944worker2 # [ 227.192058] nix-grpc-daemon[1760]: time=2026-09-22T10:39:43.836Z level=INFO msg="All 2 paths already cached"6945client # copying 1 paths to 'lb:50051'...6946client # copying path '/nix/store/77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv' to 'lb:50051'...6947worker2 # [ 227.203465] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:43Z level=debug event=rpc_start method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:561646948worker2 # [ 227.217634] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6949worker2 # [ 227.419344] nix-daemon[864]: accepted connection from pid 2049, user nix-grpc-daemon (trusted)6950worker2 # [ 227.437250] nix-daemon[864]: reaped child process 2052, status = succeeded6951worker2 # [ 227.607521] nix-daemon[864]: accepted connection from pid 2054, user nix-grpc-daemon (trusted)6952worker2 # [ 227.622865] nix-daemon[864]: reaped child process 2057, status = succeeded6953lb # [ 227.619720] niks3-server[1173]: 2026/09/22 10:39:44 INFO Received uploads request method=POST path=/api/pending_closures6954worker2 # [ 227.647883] nix-grpc-daemon[1760]: time=2026-09-22T10:39:44.291Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6955worker2 # [ 227.649485] nix-grpc-daemon[1760]: time=2026-09-22T10:39:44.292Z level=INFO msg="Uploading 77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv (760B)"6956lb # [ 227.645959] niks3-server[1173]: 2026/09/22 10:39:44 INFO Registered completed upload object_key=nar/1hrr94bvb37dwmf1giypzl9qnff72xm540ak7q49j34fyzia9ngb.nar.zst6957worker2 # [ 227.677323] nix-grpc-daemon[1760]: time=2026-09-22T10:39:44.321Z level=INFO msg="Uploading 1 narinfos"6958lb # [ 227.661413] niks3-server[1173]: 2026/09/22 10:39:44 INFO Received sign narinfos request method=POST path=/api/pending_closures/76/sign6959lb # [ 227.672535] niks3-server[1173]: 2026/09/22 10:39:44 INFO Signed narinfos id=76 count=16960lb # [ 227.678635] niks3-server[1173]: 2026/09/22 10:39:44 INFO Registered completed upload object_key=77925dzx1nz9jvb0jf5cxdn4b8gwrv5l.ls6961worker2 # [ 227.706814] nix-grpc-daemon[1760]: time=2026-09-22T10:39:44.351Z level=INFO msg="Upload complete. (472ms)"6962lb # [ 227.688911] niks3-server[1173]: 2026/09/22 10:39:44 INFO Received complete upload request method=POST path=/api/pending_closures/76/complete6963lb # [ 227.694379] niks3-server[1173]: 2026/09/22 10:39:44 INFO Completed upload id=766964worker2 # [ 227.714725] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:44Z level=info event=rpc method=AddMultipleToStore cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=0 paths=1 nar_bytes_in=7606965lb # [ 227.697859] niks3-server[1173]: 2026/09/22 10:39:44 INFO Registered completed upload object_key=77925dzx1nz9jvb0jf5cxdn4b8gwrv5l.narinfo6966worker2 # [ 227.724356] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:44Z level=debug event=rpc_start method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:561646967worker2 # [ 227.729439] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6968worker2 # [ 227.733942] nix-daemon[864]: reaped child process 2047, status = succeeded6969client # building '/nix/store/77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv' on lb:50051...6970worker2 # [ 227.745768] nix-daemon[864]: accepted connection from pid 1680, user nix-grpc-daemon (trusted)6971client # farm-top> worker2: building 77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv6972client # farm-top> disabling binary cache 'https://cache.nixos.org' for 60 seconds6973client # farm-top> LOG-farm-top-drain6974worker2 # [ 229.185644] nix-daemon[864]: reaped child process 2061, status = succeeded6975worker2 # [ 229.369360] nix-daemon[864]: accepted connection from pid 2077, user nix-grpc-daemon (trusted)6976worker2 # [ 229.387641] nix-daemon[864]: reaped child process 2080, status = succeeded6977lb # [ 229.382310] niks3-server[1173]: 2026/09/22 10:39:46 INFO Received uploads request method=POST path=/api/pending_closures6978worker2 # [ 229.410310] nix-grpc-daemon[1760]: time=2026-09-22T10:39:46.054Z level=INFO msg="Uploading 1 paths to lb (2 already cached)"6979worker2 # [ 229.413005] nix-grpc-daemon[1760]: time=2026-09-22T10:39:46.056Z level=INFO msg="Uploading bzkv277k4v9r9h4aq5h13ddcavmp36fg-farm-top (232B)"6980lb # [ 229.414584] niks3-server[1173]: 2026/09/22 10:39:46 INFO Registered completed upload object_key=log/77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv6981worker2 # [ 229.459069] nix-grpc-daemon[1760]: time=2026-09-22T10:39:46.103Z level=INFO msg="Uploading 1 narinfos"6982lb # [ 229.433750] niks3-server[1173]: 2026/09/22 10:39:46 INFO Registered completed upload object_key=nar/0l8sy8grn9pmq2n58ag0hx791zdfyxwk9319ynjqmn7yy7wl64s7.nar.zst6983lb # [ 229.446061] niks3-server[1173]: 2026/09/22 10:39:46 INFO Received sign narinfos request method=POST path=/api/pending_closures/77/sign6984lb # [ 229.456708] niks3-server[1173]: 2026/09/22 10:39:46 INFO Signed narinfos id=77 count=16985worker2 # [ 229.482390] nix-grpc-daemon[1760]: time=2026-09-22T10:39:46.126Z level=INFO msg="Upload complete. (327ms)"6986lb # [ 229.464838] niks3-server[1173]: 2026/09/22 10:39:46 INFO Registered completed upload object_key=bzkv277k4v9r9h4aq5h13ddcavmp36fg.ls6987lb # [ 229.469532] niks3-server[1173]: 2026/09/22 10:39:46 INFO Received complete upload request method=POST path=/api/pending_closures/77/complete6988lb # [ 229.474978] niks3-server[1173]: 2026/09/22 10:39:46 INFO Completed upload id=776989lb # [ 229.478189] niks3-server[1173]: 2026/09/22 10:39:46 INFO Registered completed upload object_key=bzkv277k4v9r9h4aq5h13ddcavmp36fg.narinfo6990worker2 # [ 229.493831] nix-grpc-daemon[1680]: ts=2026-09-22T10:39:46Z level=info event=rpc method=BuildDerivation cn=ci-1 peer=ipv4:192.168.1.2:56164 duration_s=1 drv=77925dzx1nz9jvb0jf5cxdn4b8gwrv5l-farm-top.drv assign_id=18 outputs=16991worker2 # [ 229.512793] nix-daemon[864]: reaped child process 2059, status = succeeded6992client: (finished: must succeed: nix build -L --store 'grpc://lb:50051?client-cert=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.pem&client-key=/nix/store/r1p0v55v6igq97giy4hh7w7h1khnwhk6-farm-certs/ci.key' --eval-store auto -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag drain >&2, in 6.49 seconds)6993client: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag drain outPath6994client: (finished: must succeed: nix eval --raw -f /nix/store/r498lgb40xdi1s9di8nh5bgv2h8vrrp1-job.nix --argstr tag drain outPath, in 0.24 seconds)6995lb # [ 229.815674] envoy[896]: conn peer=[2001:db8:1::1]:33056 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=897 tx=499 ms=300666996lb # [ 229.820930] envoy[896]: conn peer=[2001:db8:1::1]:46630 tls=TLSv1.3 subject="CN=ci-1" sni=- flags=- tls_fail="-" rx=9549 tx=4454 ms=61966997worker1: must succeed: rm /nix/.rw-store/fill6998worker1: (finished: must succeed: rm /nix/.rw-store/fill, in 0.15 seconds)6999worker1: waiting for success: journalctl -u nix-grpc-daemon -o cat | grep -q 'event=healthy reason=min_free'7000worker1 # [ 230.895967] nix-grpc-daemon[1628]: ts=2026-09-22T10:39:47Z level=info event=healthy reason=min_free7001worker1: (finished: waiting for success: journalctl -u nix-grpc-daemon -o cat | grep -q 'event=healthy reason=min_free', in 1.12 seconds)7002(finished: subtest: low disk drains a worker and builds go to the other, in 9.27 seconds)7003(finished: run the VM test script, in 232.04 seconds)7004test script finished in 232.34s7005cleanup7006kill QemuMachine (pid 48)7007client # qemu-system-aarch64: terminating on signal 15 from pid 39 (/nix/store/lfh101cfjj8p8npcari9hk0lbqi33yvh-python3-3.14.7/bin/python3.14)7008kill QemuMachine (pid 50)7009lb # qemu-system-aarch64: terminating on signal 15 from pid 39 (/nix/store/lfh101cfjj8p8npcari9hk0lbqi33yvh-python3-3.14.7/bin/python3.14)7010kill QemuMachine (pid 51)7011worker1 # qemu-system-aarch64: terminating on signal 15 from pid 39 (/nix/store/lfh101cfjj8p8npcari9hk0lbqi33yvh-python3-3.14.7/bin/python3.14)7012kill QemuMachine (pid 52)7013worker2 # qemu-system-aarch64: terminating on signal 15 from pid 39 (/nix/store/lfh101cfjj8p8npcari9hk0lbqi33yvh-python3-3.14.7/bin/python3.14)7014(finished: cleanup, in 1.07 seconds)7015additionally exposed symbols:7016 client, lb, worker1, worker2,7017 vlan1,7018 start_all, test_script, machines, machines_qemu, machines_nspawn, vlans, driver, log, os, create_machine, subtest, run_tests, join_all, retry, serial_stdout_off, serial_stdout_on, polling_condition, BaseMachine, QemuMachine, NspawnMachine, t, debug, dump_machine_ssh