Mock Version: 5.0
Mock Version: 5.0
Mock Version: 5.0
ENTER ['do_with_status'](['bash', '--login', '-c', '/usr/bin/rpmbuild -bs --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'], chrootPath='/var/lib/mock/f39-build-1926322-43041/root'env={'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;<mock-chroot>\\007"', 'PS1': '<mock-chroot> \\s-\\v\\$ ', 'LANG': 'C.UTF-8'}shell=Falselogger=<mockbuild.trace_decorator.getLog object at 0xffffff719f4b10>timeout=864000uid=996gid=135user='mockbuild'nspawn_args=[]unshare_net=TrueprintOutput=False)
Executing command: ['bash', '--login', '-c', '/usr/bin/rpmbuild -bs --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'] with env {'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;<mock-chroot>\\007"', 'PS1': '<mock-chroot> \\s-\\v\\$ ', 'LANG': 'C.UTF-8'} and shell False
Building target platforms: noarch
Building for target noarch
setting SOURCE_DATE_EPOCH=1700697600
Wrote: /builddir/build/SRPMS/copr-backend-1.173-1.fc39.src.rpm
Child return code was: 0
ENTER ['do_with_status'](['bash', '--login', '-c', '/usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'], chrootPath='/var/lib/mock/f39-build-1926322-43041/root'env={'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;<mock-chroot>\\007"', 'PS1': '<mock-chroot> \\s-\\v\\$ ', 'LANG': 'C.UTF-8'}shell=Falselogger=<mockbuild.trace_decorator.getLog object at 0xffffff719f4b10>timeout=864000uid=996gid=135user='mockbuild'nspawn_args=[]unshare_net=TrueprintOutput=False)
Executing command: ['bash', '--login', '-c', '/usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'] with env {'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;<mock-chroot>\\007"', 'PS1': '<mock-chroot> \\s-\\v\\$ ', 'LANG': 'C.UTF-8'} and shell False
Building target platforms: noarch
Building for target noarch
setting SOURCE_DATE_EPOCH=1700697600
Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.gCxU22
+ umask 022
+ cd /builddir/build/BUILD
+ cd /builddir/build/BUILD
+ rm -rf copr-backend-1.173
+ /usr/lib/rpm/rpmuncompress -x /builddir/build/SOURCES/copr-backend-1.173.tar.gz
+ STATUS=0
+ '[' 0 -ne 0 ']'
+ cd copr-backend-1.173
+ rm -rf /builddir/build/BUILD/copr-backend-1.173-SPECPARTS
+ /usr/bin/mkdir -p /builddir/build/BUILD/copr-backend-1.173-SPECPARTS
+ /usr/lib/rpm/rpmuncompress -x /builddir/build/SOURCES/test-data-copr-backend-5.tar.gz
+ STATUS=0
+ '[' 0 -ne 0 ']'
+ /usr/bin/chmod -Rf a+rX,u+w,g-w,o-w .
+ RPM_EC=0
++ jobs -p
+ exit 0
Executing(%build): /bin/sh -e /var/tmp/rpm-tmp.7mknUR
+ umask 022
+ cd /builddir/build/BUILD
+ CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ export CFLAGS
+ CXXFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ export CXXFLAGS
+ FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules '
+ export FFLAGS
+ FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules '
+ export FCFLAGS
+ VALAFLAGS=-g
+ export VALAFLAGS
+ RUSTFLAGS='-Copt-level=3 -Cdebuginfo=2 -Ccodegen-units=1 -Cstrip=none -Cforce-frame-pointers=yes -Clink-arg=-Wl,-z,relro -Clink-arg=-Wl,-z,now --cap-lints=warn'
+ export RUSTFLAGS
+ LDFLAGS='-Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -Wl,--build-id=sha1  '
+ export LDFLAGS
+ LT_SYS_LIBRARY_PATH=/usr/lib:
+ export LT_SYS_LIBRARY_PATH
+ CC=gcc
+ export CC
+ CXX=g++
+ export CXX
+ cd copr-backend-1.173
+ make -C docs -j8 html
make: Entering directory '/builddir/build/BUILD/copr-backend-1.173/docs'
sphinx-build-3 -b html -d build/doctrees   source build/html
Running Sphinx v6.2.1
making output directory... done
building [mo]: targets for 0 po files that are out of date
writing output... 
building [html]: targets for 32 source files that are out of date
updating environment: [new config] 32 added, 0 changed, 0 removed
reading sources... [  3%] Autodoc
reading sources... [  6%] Backend_design
reading sources... [  9%] Configuration
reading sources... [ 12%] INSTALL
reading sources... [ 15%] README
reading sources... [ 18%] Statistics
reading sources... [ 21%] VmManagement
reading sources... [ 25%] index
reading sources... [ 28%] package/actions
reading sources... [ 31%] package/constants
reading sources... [ 34%] package/createrepo
reading sources... [ 37%] package/daemons/backend
reading sources... [ 40%] package/daemons/dispatcher
reading sources... [ 43%] package/daemons/job_grab
reading sources... [ 46%] package/daemons/log
reading sources... [ 50%] package/daemons/vm_master
reading sources... [ 53%] package/exceptions
reading sources... [ 56%] package/frontend
reading sources... [ 59%] package/helpers
reading sources... [ 62%] package/job
reading sources... [ 65%] package/mockremote/__init__
reading sources... [ 68%] package/mockremote/builder
reading sources... [ 71%] package/sign
reading sources... [ 75%] package/vm_manage/__init__
reading sources... [ 78%] package/vm_manage/check
reading sources... [ 81%] package/vm_manage/event_handle
reading sources... [ 84%] package/vm_manage/executor
reading sources... [ 87%] package/vm_manage/manager
reading sources... [ 90%] package/vm_manage/models
reading sources... [ 93%] package/vm_manage/spawn
reading sources... [ 96%] package/vm_manage/terminate
reading sources... [100%] run_scripts
WARNING: autodoc: failed to import module 'actions' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'constants' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'createrepo' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'daemons.backend' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'daemons.dispatcher' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'daemons.job_grab' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'daemons.log' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'daemons.vm_master' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'exceptions' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'frontend' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'helpers' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'job' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'mockremote.__init__' from module 'backend'; the following exception was raised:
No module named 'backend'
looking for now-outdated files... none found
pickling environment... WARNING: autodoc: failed to import module 'mockremote.builder' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'sign' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.__init__' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.check' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.event_handle' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.executor' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.manager' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.models' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.spawn' from module 'backend'; the following exception was raised:
No module named 'backend'
WARNING: autodoc: failed to import module 'vm_manage.terminate' from module 'backend'; the following exception was raised:
No module named 'backend'
done
checking consistency... done
preparing documents... done
writing output... [  3%] Autodoc
writing output... [  6%] Backend_design
writing output... [  9%] Configuration
writing output... [ 12%] INSTALL
writing output... [ 15%] README
writing output... [ 18%] Statistics
writing output... [ 21%] VmManagement
writing output... [ 25%] index
writing output... [ 28%] package/actions
writing output... [ 31%] package/constants
writing output... [ 34%] package/createrepo
writing output... [ 37%] package/daemons/backend
writing output... [ 40%] package/daemons/dispatcher
writing output... [ 43%] package/daemons/job_grab
writing output... [ 46%] package/daemons/log
writing output... [ 50%] package/daemons/vm_master
writing output... [ 53%] package/exceptions
writing output... [ 56%] package/frontend
writing output... [ 59%] package/helpers
writing output... [ 62%] package/job
writing output... [ 65%] package/mockremote/__init__
writing output... [ 68%] package/mockremote/builder
writing output... [ 71%] package/sign
writing output... [ 75%] package/vm_manage/__init__
writing output... [ 78%] package/vm_manage/check
writing output... [ 81%] package/vm_manage/event_handle
writing output... [ 84%] package/vm_manage/executor
writing output... [ 87%] package/vm_manage/manager
writing output... [ 90%] package/vm_manage/models
writing output... [ 93%] package/vm_manage/spawn
writing output... [ 96%] package/vm_manage/terminate
writing output... [100%] run_scripts
generating indices... genindex done
highlighting module code... 
writing additional pages... search done
copying static files... done
copying extra files... done
dumping search index in English (code: en)... done
dumping object inventory... done
build succeeded, 23 warnings.
The HTML pages are in build/html.
Build finished. The HTML pages are in build/html.
make: Leaving directory '/builddir/build/BUILD/copr-backend-1.173/docs'
+ CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ LDFLAGS='-Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -Wl,--build-id=sha1  '
+ /usr/bin/python3 setup.py build '--executable=/usr/bin/python3 -sP'
running build
running build_py
creating build
creating build/lib
creating build/lib/copr_backend
copying copr_backend/setup.py -> build/lib/copr_backend
copying copr_backend/hitcounter.py -> build/lib/copr_backend
copying copr_backend/dispatcher.py -> build/lib/copr_backend
copying copr_backend/background_worker_build.py -> build/lib/copr_backend
copying copr_backend/app.py -> build/lib/copr_backend
copying copr_backend/sshcmd.py -> build/lib/copr_backend
copying copr_backend/cancellable_thread.py -> build/lib/copr_backend
copying copr_backend/createrepo.py -> build/lib/copr_backend
copying copr_backend/exceptions.py -> build/lib/copr_backend
copying copr_backend/frontend.py -> build/lib/copr_backend
copying copr_backend/constants.py -> build/lib/copr_backend
copying copr_backend/vm_alloc.py -> build/lib/copr_backend
copying copr_backend/worker_manager.py -> build/lib/copr_backend
copying copr_backend/rpm_builds.py -> build/lib/copr_backend
copying copr_backend/__init__.py -> build/lib/copr_backend
copying copr_backend/helpers.py -> build/lib/copr_backend
copying copr_backend/ans_utils.py -> build/lib/copr_backend
copying copr_backend/background_worker.py -> build/lib/copr_backend
copying copr_backend/msgbus.py -> build/lib/copr_backend
copying copr_backend/job.py -> build/lib/copr_backend
copying copr_backend/sign.py -> build/lib/copr_backend
copying copr_backend/actions.py -> build/lib/copr_backend
creating build/lib/copr_backend/stats_templates
copying copr_backend/stats_templates/__init__.py -> build/lib/copr_backend/stats_templates
creating build/lib/copr_backend/daemons
copying copr_backend/daemons/action_dispatcher.py -> build/lib/copr_backend/daemons
copying copr_backend/daemons/build_dispatcher.py -> build/lib/copr_backend/daemons
copying copr_backend/daemons/__init__.py -> build/lib/copr_backend/daemons
copying copr_backend/daemons/log.py -> build/lib/copr_backend/daemons
running egg_info
creating copr_backend.egg-info
writing copr_backend.egg-info/PKG-INFO
writing dependency_links to copr_backend.egg-info/dependency_links.txt
writing top-level names to copr_backend.egg-info/top_level.txt
writing manifest file 'copr_backend.egg-info/SOURCES.txt'
reading manifest file 'copr_backend.egg-info/SOURCES.txt'
adding license file 'LICENSE'
writing manifest file 'copr_backend.egg-info/SOURCES.txt'
copying copr_backend/stats_templates/arches.html.j2 -> build/lib/copr_backend/stats_templates
copying copr_backend/stats_templates/chroots.html.j2 -> build/lib/copr_backend/stats_templates
copying copr_backend/stats_templates/distro.html.j2 -> build/lib/copr_backend/stats_templates
copying copr_backend/stats_templates/index.html.j2 -> build/lib/copr_backend/stats_templates
copying copr_backend/stats_templates/topmost.html.j2 -> build/lib/copr_backend/stats_templates
+ RPM_EC=0
++ jobs -p
+ exit 0
Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.UXwNau
+ umask 022
+ cd /builddir/build/BUILD
+ '[' /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch '!=' / ']'
+ rm -rf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch
++ dirname /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch
+ mkdir -p /builddir/build/BUILDROOT
+ mkdir /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch
+ CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ export CFLAGS
+ CXXFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ export CXXFLAGS
+ FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules '
+ export FFLAGS
+ FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules '
+ export FCFLAGS
+ VALAFLAGS=-g
+ export VALAFLAGS
+ RUSTFLAGS='-Copt-level=3 -Cdebuginfo=2 -Ccodegen-units=1 -Cstrip=none -Cforce-frame-pointers=yes -Clink-arg=-Wl,-z,relro -Clink-arg=-Wl,-z,now --cap-lints=warn'
+ export RUSTFLAGS
+ LDFLAGS='-Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -Wl,--build-id=sha1  '
+ export LDFLAGS
+ LT_SYS_LIBRARY_PATH=/usr/lib:
+ export LT_SYS_LIBRARY_PATH
+ CC=gcc
+ export CC
+ CXX=g++
+ export CXX
+ cd copr-backend-1.173
+ CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ LDFLAGS='-Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -Wl,--build-id=sha1  '
+ /usr/bin/python3 setup.py install -O1 --skip-build --root /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch --prefix /usr
running install
/usr/lib/python3.12/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated.
!!
        ********************************************************************************
        Please avoid running ``setup.py`` directly.
        Instead, use pypa/build, pypa/installer, pypa/build or
        other standards-based tools.
        Follow the current Python packaging guidelines when building
        Python RPM packages.
        See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html
        and https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/ for details.
        ********************************************************************************
!!
  self.initialize_options()
running install_lib
creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr
creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib
creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12
creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages
creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/setup.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates
copying build/lib/copr_backend/stats_templates/index.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates
copying build/lib/copr_backend/stats_templates/chroots.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates
copying build/lib/copr_backend/stats_templates/topmost.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates
copying build/lib/copr_backend/stats_templates/distro.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates
copying build/lib/copr_backend/stats_templates/arches.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates
copying build/lib/copr_backend/stats_templates/__init__.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates
creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons
copying build/lib/copr_backend/daemons/action_dispatcher.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons
copying build/lib/copr_backend/daemons/build_dispatcher.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons
copying build/lib/copr_backend/daemons/__init__.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons
copying build/lib/copr_backend/daemons/log.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons
copying build/lib/copr_backend/hitcounter.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/dispatcher.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/background_worker_build.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/app.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/sshcmd.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/cancellable_thread.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/createrepo.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/exceptions.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/frontend.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/constants.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/vm_alloc.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/worker_manager.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/rpm_builds.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/__init__.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/helpers.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/ans_utils.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/background_worker.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/msgbus.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/job.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/sign.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
copying build/lib/copr_backend/actions.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/setup.py to setup.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates/__init__.py to __init__.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/action_dispatcher.py to action_dispatcher.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/build_dispatcher.py to build_dispatcher.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/__init__.py to __init__.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/log.py to log.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/hitcounter.py to hitcounter.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/dispatcher.py to dispatcher.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/background_worker_build.py to background_worker_build.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/app.py to app.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/sshcmd.py to sshcmd.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/cancellable_thread.py to cancellable_thread.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/createrepo.py to createrepo.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/exceptions.py to exceptions.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/frontend.py to frontend.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/constants.py to constants.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/vm_alloc.py to vm_alloc.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/worker_manager.py to worker_manager.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/rpm_builds.py to rpm_builds.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/__init__.py to __init__.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/helpers.py to helpers.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/ans_utils.py to ans_utils.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/background_worker.py to background_worker.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/msgbus.py to msgbus.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/job.py to job.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/sign.py to sign.cpython-312.pyc
byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/actions.py to actions.cpython-312.pyc
writing byte-compilation script '/tmp/tmp9w41tdf3.py'
/usr/bin/python3 /tmp/tmp9w41tdf3.py
removing /tmp/tmp9w41tdf3.py
running install_egg_info
running egg_info
writing copr_backend.egg-info/PKG-INFO
writing dependency_links to copr_backend.egg-info/dependency_links.txt
writing top-level names to copr_backend.egg-info/top_level.txt
reading manifest file 'copr_backend.egg-info/SOURCES.txt'
adding license file 'LICENSE'
writing manifest file 'copr_backend.egg-info/SOURCES.txt'
Copying copr_backend.egg-info to /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend-1.173-py3.12.egg-info
running install_scripts
+ rm -rfv /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/bin/__pycache__
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/var/lib/copr/public_html/results
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/lighttpd/
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/copr
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/logrotate.d/
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/systemd/system
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//var/log/copr-backend
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//var/run/copr-backend/
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/lib/tmpfiles.d
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/sbin
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.daily
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.weekly
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/sudoers.d
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/bin/
+ cp -a copr-backend-service /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/sbin/
+ cp -a run/copr-assure-permissions run/copr-aws-s3-hitcounter run/copr-backend-analyze-results run/copr-backend-generate-graphs run/copr-backend-process-action run/copr-backend-process-build run/copr-backend-resultdir-cleaner run/copr-backend-unknown-resalloc-tickets.py run/copr-compress-live-logs run/copr-rename-chroot run/copr-repo run/copr-run-dispatcher run/copr_find_obsolete_builds.sh run/copr_find_wrong_chroot_artifacts.py run/copr_fix_gpg.py run/copr_log_hitcounter.py run/copr_print_results_to_delete.py run/copr_prune_results.py run/copr_prune_srpms.py run/copr_run_logger.py run/copr_sign_unsigned.py run/print_queues.py /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/bin/
+ cp -a conf/copr-be.conf.example /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/copr/copr-be.conf
+ install -p -m 755 conf/crontab/daily /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.daily/copr-backend
+ install -p -m 755 conf/crontab/weekly /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.weekly/copr-backend
+ cp -a conf/lighttpd/dirlisting.conf conf/lighttpd/mime.conf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/lighttpd/
+ cp -a conf/logrotate/copr-backend /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/logrotate.d/
+ cp -a conf/tmpfiles.d/copr-backend.conf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/lib/tmpfiles.d
+ touch /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/var/log/copr-backend/copr.log
+ touch /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/var/log/copr-backend/prune_old.log
+ cp -a units/copr-backend.target units/copr-backend-action.service units/copr-backend-build.service units/copr-backend-log.service /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/lib/systemd/system/
+ install -m 0644 conf/copr.sudoers.d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/sudoers.d/copr
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/logstash.d
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/logstash/patterns/
+ cp -a conf/logstash/lighttpd.pattern /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/logstash/patterns/lighttpd.pattern
+ install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/examples//etc/logstash.d
+ cp -a conf/logstash/copr_backend.conf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/examples//etc/logstash.d/copr_backend.conf
+ cp -a docs/build/html /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/
+ /usr/bin/find-debuginfo -j8 --strict-build-id -m -i --build-id-seed 1.173-1.fc39 --unique-debug-suffix -1.173-1.fc39.noarch --unique-debug-src-base copr-backend-1.173-1.fc39.noarch --run-dwz --dwz-low-mem-die-limit 10000000 --dwz-max-die-limit 50000000 -S debugsourcefiles.list /builddir/build/BUILD/copr-backend-1.173
find-debuginfo: starting
Extracting debug info from 0 files
Creating .debug symlinks for symlinks to ELF files
find: ‘debug’: No such file or directory
find-debuginfo: done
+ /usr/lib/rpm/check-buildroot
+ /usr/lib/rpm/redhat/brp-ldconfig
+ /usr/lib/rpm/brp-compress
+ /usr/lib/rpm/redhat/brp-strip-lto /usr/bin/strip
+ /usr/lib/rpm/brp-strip-static-archive /usr/bin/strip
+ /usr/lib/rpm/check-rpaths
+ /usr/lib/rpm/redhat/brp-mangle-shebangs
mangling shebang in /usr/bin/copr_find_obsolete_builds.sh from /bin/bash to #!/usr/bin/bash
mangling shebang in /usr/bin/copr-compress-live-logs from /bin/sh to #!/usr/bin/sh
mangling shebang in /usr/sbin/copr-backend-service from /bin/sh to #!/usr/bin/sh
mangling shebang in /etc/cron.weekly/copr-backend from /bin/sh to #!/usr/bin/sh
+ /usr/lib/rpm/brp-remove-la-files
+ env /usr/lib/rpm/redhat/brp-python-bytecompile '' 1 0 -j8
Bytecompiling .py files below /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12 using python3.12
+ /usr/lib/rpm/redhat/brp-python-hardlink
Executing(%check): /bin/sh -e /var/tmp/rpm-tmp.ZtsnT6
+ umask 022
+ cd /builddir/build/BUILD
+ CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ export CFLAGS
+ CXXFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer '
+ export CXXFLAGS
+ FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules '
+ export FFLAGS
+ FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules '
+ export FCFLAGS
+ VALAFLAGS=-g
+ export VALAFLAGS
+ RUSTFLAGS='-Copt-level=3 -Cdebuginfo=2 -Ccodegen-units=1 -Cstrip=none -Cforce-frame-pointers=yes -Clink-arg=-Wl,-z,relro -Clink-arg=-Wl,-z,now --cap-lints=warn'
+ export RUSTFLAGS
+ LDFLAGS='-Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -Wl,--build-id=sha1  '
+ export LDFLAGS
+ LT_SYS_LIBRARY_PATH=/usr/lib:
+ export LT_SYS_LIBRARY_PATH
+ CC=gcc
+ export CC
+ CXX=g++
+ export CXX
+ cd copr-backend-1.173
+ ./run_tests.sh -vv --no-cov
+ set -e
++ dirname ./run_tests.sh
+ srcdir=.
++ grep -E '%global[[:space:]]*tests_version'
++ awk '{ print $3 }'
+ test_tarball_version=5
++ grep -E '%global[[:space:]]*tests_tar'
++ awk '{ print $3 }'
+ test_tarball_name=test-data-copr-backend
+ test_tarball_extracted=test-data-copr-backend-5
+ test_tarball=test-data-copr-backend-5.tar.gz
+ test -d test-data-copr-backend-5
+ export TEST_DATA_DIRECTORY
++ readlink -f test-data-copr-backend-5
+ TEST_DATA_DIRECTORY=/builddir/build/BUILD/copr-backend-1.173/test-data-copr-backend-5
+ REDIS_PORT=7777
+ trap cleanup EXIT
+ redis-server --port 7777
++ readlink -f ../common
+ common_path=/builddir/build/BUILD/common
++ readlink -f ../messaging
+ messaging_path=/builddir/build/BUILD/messaging
+ export PYTHONPATH=/builddir/build/BUILD/common:/builddir/build/BUILD/messaging:/builddir/build/BUILD/copr-backend-1.173:/builddir/build/BUILD/copr-backend-1.173/tests:/builddir/build/BUILD/copr-backend-1.173/run
+ PYTHONPATH=/builddir/build/BUILD/common:/builddir/build/BUILD/messaging:/builddir/build/BUILD/copr-backend-1.173:/builddir/build/BUILD/copr-backend-1.173/tests:/builddir/build/BUILD/copr-backend-1.173/run
+ export PATH=/builddir/build/BUILD/copr-backend-1.173/run:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/sbin
+ PATH=/builddir/build/BUILD/copr-backend-1.173/run:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/sbin
+ COVPARAMS=(--cov-report term-missing --cov ./copr_backend --cov ./run)
+ KEEP_ARGS=()
+ for arg in "$@"
+ case $arg in
+ KEEP_ARGS+=("$arg")
+ for arg in "$@"
+ case $arg in
+ COVPARAMS=()
+ python3 -m pytest -s tests -vv
============================= test session starts ==============================
platform linux -- Python 3.12.0, pytest-7.3.2, pluggy-1.2.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /builddir/build/BUILD/copr-backend-1.173
collecting ... collected 187 items
tests/test_action.py::TestAction::test_action_run_legal_flag PASSED
tests/test_action.py::TestAction::test_action_handle_forks PASSED
tests/test_action.py::TestAction::test_action_run_rename SKIPPED (Fixme, test doesn't work.)
tests/test_action.py::TestAction::test_action_run_rename_success_on_empty_src SKIPPED (Fixme, test doesn't work.)
tests/test_action.py::TestAction::test_action_run_rename_failure_on_destination_exists SKIPPED (Fixme, test doesn't work.)
tests/test_action.py::TestAction::test_action_run_delete_copr PASSED
tests/test_action.py::TestAction::test_action_run_delete_copr_remove_folders SKIPPED (Fixme, test doesn't work.)
tests/test_action.py::TestAction::test_delete_no_chroot_dirs PASSED
tests/test_action.py::TestAction::test_delete_build_succeeded PASSED
tests/test_action.py::TestAction::test_delete_build_acr_reflected[False] PASSED
tests/test_action.py::TestAction::test_delete_build_acr_reflected[True] PASSED
tests/test_action.py::TestAction::test_delete_build_succeeded_createrepo_error PASSED
tests/test_action.py::TestAction::test_delete_two_chroots PASSED
tests/test_action.py::TestAction::test_delete_two_chroots_two_remain PASSED
tests/test_action.py::TestAction::test_delete_build_with_bad_pkg_name PASSED
tests/test_action.py::TestAction::test_delete_multiple_builds_succeeded PASSED
tests/test_action.py::TestAction::test_handle_createrepo_ok[False] PASSED
tests/test_action.py::TestAction::test_handle_createrepo_ok[True] PASSED
tests/test_action.py::TestAction::test_handle_createrepo_failure_1 PASSED
tests/test_action.py::TestAction::test_handle_generate_gpg_key SKIPPED (Fixme, test doesn't work.)
tests/test_action.py::TestAction::test_request_exception_is_taken_care_of_when_posting_to_frontend SKIPPED (Fixme, test doesn't work.)
tests/test_action.py::TestAction::test_delete_chroot PASSED
tests/test_action.py::TestAction::test_comps_create PASSED
tests/test_action.py::TestAction::test_remove_dirs PASSED
tests/test_background_worker_build.py::test_waiting_for_repo_fail PASSED
tests/test_background_worker_build.py::test_waiting_for_repo_success PASSED
tests/test_background_worker_build.py::test_full_rpm_build_no_sign PASSED
tests/test_background_worker_build.py::test_prev_build_backup PASSED
tests/test_background_worker_build.py::test_full_srpm_build PASSED
tests/test_background_worker_build.py::test_build_and_sign PASSED
tests/test_background_worker_build.py::test_sign_built_packages_exception PASSED
tests/test_background_worker_build.py::test_unexpected_exception PASSED
tests/test_background_worker_build.py::test_build_info_file_failure PASSED
tests/test_background_worker_build.py::test_invalid_job_info PASSED
tests/test_background_worker_build.py::test_cancel_build_on_vm_allocation PASSED
tests/test_background_worker_build.py::test_cancel_build_on_tail_log_no_ssh PASSED
tests/test_background_worker_build.py::test_cancel_before_vm PASSED
tests/test_background_worker_build.py::test_cancel_before_start PASSED
tests/test_background_worker_build.py::test_build_retry PASSED
tests/test_background_worker_build.py::test_fe_disallowed_start PASSED
tests/test_background_worker_build.py::test_fe_failed_start PASSED
tests/test_background_worker_build.py::test_cancel_script_failure PASSED
tests/test_background_worker_build.py::test_cancel_build_during_log_download PASSED
tests/test_background_worker_build.py::test_ssh_connection_error PASSED
tests/test_background_worker_build.py::test_average_step PASSED
tests/test_background_worker_build.py::test_retry_for_ssh_tail_failure PASSED
tests/test_background_worker_build.py::test_build_failure PASSED
tests/test_background_worker_build.py::test_createrepo_failure PASSED
tests/test_background_worker_build.py::test_existing_compressed_file PASSED
tests/test_background_worker_build.py::test_tail_f_nonzero_exit PASSED
tests/test_background_worker_build.py::test_wrong_copr_rpmbuild_daemon_output PASSED
tests/test_background_worker_build.py::test_unable_to_start_builder PASSED
tests/test_background_worker_build.py::test_retry_vm_factory_take PASSED
tests/test_background_worker_build.py::test_failed_build_retry PASSED
tests/test_background_worker_build.py::test_buildjob_tags PASSED
tests/test_background_worker_build.py::test_task_url PASSED
tests/test_build_dispatcher.py::test_priority_numbers PASSED
tests/test_build_dispatcher.py::test_frontend_priority[True-2000000] PASSED
tests/test_build_dispatcher.py::test_frontend_priority[False-0] PASSED
tests/test_build_dispatcher.py::test_sandbox_priority PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_minimal_file_and_defaults PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_correct_build_limits PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_invalid_limits[builds_max_workers=asdfa\n] PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_invalid_limits[actions_max_workers=asdfa\n] PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc\n] PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc=asdf\n] PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc=10=\n] PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc=1,\n] PASSED
tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_tag=abc=1,\n] PASSED
tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_normal PASSED
tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_already_done PASSED
tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_other_already_done PASSED
tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_devel_mismatch PASSED
tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_full_we_take_others PASSED
tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_full_others_take_us PASSED
tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_task_limit PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend[get] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend[post] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend[put] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_wrappers[get] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_wrappers[post] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_wrappers[put] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_first_try_ok PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_second_try_ok[get] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_second_try_ok[post] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_second_try_ok[put] PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_err_400 PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_all_attempts_failed PASSED
tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_indefinitely PASSED
tests/test_frontend.py::TestFrontendClient::test_retries_on_outdated_frontend PASSED
tests/test_frontend.py::TestFrontendClient::test_update PASSED
tests/test_frontend.py::TestFrontendClient::test_starting_build PASSED
tests/test_frontend.py::TestFrontendClient::test_starting_build_err PASSED
tests/test_frontend.py::TestFrontendClient::test_starting_build_err_2 PASSED
tests/test_frontend.py::TestFrontendClient::test_reschedule_build PASSED
tests/test_helpers.py::TestHelpers::test_redis_logger_exception PASSED
tests/test_helpers.py::TestHelpers::test_get_chroot_arch PASSED
tests/test_helpers.py::TestHelpers::test_format_filename PASSED
tests/test_helpers.py::TestHelpers::test_walk_limited PASSED
tests/test_helpers.py::TestHelpers::test_recursive_copy_and_link_rpms PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks FAILED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_add_subdir PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_createrepo PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_already_processed PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_two_builds PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_full PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_others_full PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_add_del_mixup PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_add_subdir_devel PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add0] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add1] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add2] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add3] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_none_doesnt_raise PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_do_stat PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[True-fedora-37---no-database] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[False-centos-stream-8---no-database] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[True-rhel-8---no-database] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[False-epel-8---no-database] PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_el5 SKIPPED (createrepo_c dropped md5 checksum support)
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_noop PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_in_call PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_passes PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_passes_2 PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_passes_3 PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_comps_present PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_run_prunerepo SKIPPED (Works locally and in Copr but fails in Koji)
tests/test_modifyrepo.py::TestModifyRepo::test_pruner_arg_max PASSED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] FAILED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] ERROR
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED
tests/test_modifyrepo.py::test_aws_cdn_refresh PASSED
tests/test_sign.py::TestSign::test_get_pubkey PASSEDFAILED
tests/test_sign.py::TestSign::test_get_pubkey_error PASSED
tests/test_sign.py::TestSign::test_get_pubkey_unknown_key PASSED
tests/test_sign.py::TestSign::test_get_pubkey_unknown_error PASSED
tests/test_sign.py::TestSign::test_get_pubkey_outfile PASSED
tests/test_sign.py::TestSign::test_sign_one 
tests/test_modifyrepo.py::test_aws_cdn_refresh PASSED
tests/test_sign.py::TestSign::test_sign_one_popen_error PASSED
tests/test_sign.py::TestSign::test_sign_one_cmd_erro PASSED
tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly PASSED
tests/test_sign.py::TestSign::test_get_pubkey PASSED
tests/test_sign.py::TestSign::test_get_pubkey_error PASSED
tests/test_sign.py::TestSign::test_get_pubkey_unknown_key PASSED
tests/test_sign.py::TestSign::test_get_pubkey_unknown_error PASSED
tests/test_sign.py::TestSign::test_get_pubkey_outfile PASSED
tests/test_sign.py::TestSign::test_sign_one PASSED
tests/test_sign.py::TestSign::test_sign_one_popen_error PASSED
tests/test_sign.py::TestSign::test_sign_one_cmd_erro PASSED
tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly PASSED
tests/test_sign.py::TestSign::test_create_user_keys PASSED
tests/test_sign.py::TestSign::test_create_user_keys_error_1 PASSED
tests/test_sign.py::TestSign::test_create_user_keys_err PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all PASSED
tests/test_sign.py::test_chroot_gpg_hashes PASSED
tests/test_sshcmd.py::test_ipv4_ipv6_rsync PASSED
tests/test_vm_alloc.py::test_ticket PASSED
tests/test_vm_alloc.py::test_ticket_with_args PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_raises FAILEDPASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSED
tests/test_sign.py::TestSign::test_create_user_keys PASSED
tests/test_sign.py::TestSign::test_create_user_keys_error_1 PASSED
tests/test_sign.py::TestSign::test_create_user_keys_err 
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] ERROR
tests/test_modifyrepo.py::test_aws_cdn_refresh PASSEDPASSED
tests/test_worker_limits.py::test_predicate_worker_limit 
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSEDPASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok 
tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSEDPASSED
tests/test_worker_limits.py::test_group_worker_limit PASSEDPASSED
tests/test_sign.py::TestSign::test_get_pubkey 
tests/test_worker_limits.py::test_worker_limit_info 
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey PASSEDPASSED
tests/test_worker_limits.py::test_string_counter 
tests/test_sign.py::TestSign::test_get_pubkey_error PASSED
tests/test_worker_manager.py::TestPrioQueue::test_queue_order PASSEDPASSEDPASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key 
tests/test_sign.py::TestSign::test_get_pubkey_unknown_key 
tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSED
tests/test_worker_manager.py::TestPrioQueue::test_push_back PASSED
tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSEDPASSEDPASSED
tests/test_sign.py::TestSign::test_get_pubkey_unknown_error 
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one 
tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected PASSEDPASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all 
tests/test_sign.py::TestSign::test_get_pubkey_outfile PASSEDPASSED
tests/test_sign.py::test_chroot_gpg_hashes 
tests/test_sign.py::TestSign::test_sign_one PASSEDPASSED
tests/test_sshcmd.py::test_ipv4_ipv6_rsync 
tests/test_sign.py::TestSign::test_sign_one_popen_error PASSED
tests/test_vm_alloc.py::test_ticket PASSED
tests/test_sign.py::TestSign::test_sign_one_cmd_erro PASSED
tests/test_vm_alloc.py::test_ticket_with_args PASSEDPASSED
tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly 
tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_raises PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSED
tests/test_worker_limits.py::test_predicate_worker_limit PASSED
tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSED
tests/test_worker_limits.py::test_group_worker_limit PASSED
tests/test_worker_limits.py::test_worker_limit_info PASSED
tests/test_worker_limits.py::test_string_counter PASSED
tests/test_worker_manager.py::TestPrioQueue::test_queue_order PASSED
tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSED
tests/test_worker_manager.py::TestPrioQueue::test_push_back PASSED
tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSED
tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected PASSED
tests/test_sign.py::TestSign::test_create_user_keys PASSED
tests/test_sign.py::TestSign::test_create_user_keys_error_1 PASSED
tests/test_sign.py::TestSign::test_create_user_keys_err PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key PASSEDFAILED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all PASSED
tests/test_sign.py::test_chroot_gpg_hashes 
tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] ERRORPASSED
tests/test_modifyrepo.py::test_aws_cdn_refresh 
tests/test_sshcmd.py::test_ipv4_ipv6_rsync PASSED
tests/test_vm_alloc.py::test_ticket PASSEDPASSED
tests/test_vm_alloc.py::test_ticket_with_args 
tests/test_sign.py::TestSign::test_get_pubkey PASSEDPASSED
tests/test_sign.py::TestSign::test_get_pubkey_error 
tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSED
tests/test_sign.py::TestSign::test_get_pubkey_unknown_key PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_raises PASSED
tests/test_sign.py::TestSign::test_get_pubkey_unknown_error PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSEDPASSED
tests/test_sign.py::TestSign::test_get_pubkey_outfile 
tests/test_worker_limits.py::test_predicate_worker_limit PASSED
tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSED
tests/test_worker_limits.py::test_group_worker_limit PASSEDPASSED
tests/test_worker_limits.py::test_worker_limit_info 
tests/test_sign.py::TestSign::test_sign_one PASSEDPASSED
tests/test_sign.py::TestSign::test_sign_one_popen_error 
tests/test_worker_limits.py::test_string_counter PASSEDPASSED
tests/test_sign.py::TestSign::test_sign_one_cmd_erro PASSED
tests/test_worker_manager.py::TestPrioQueue::test_queue_order 
tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly PASSED
tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSED
tests/test_worker_manager.py::TestPrioQueue::test_push_back PASSED
tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSED
tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected PASSED
tests/test_sign.py::TestSign::test_create_user_keys PASSED
tests/test_sign.py::TestSign::test_create_user_keys_error_1 PASSED
tests/test_sign.py::TestSign::test_create_user_keys_err PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one PASSED
tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all PASSED
tests/test_sign.py::test_chroot_gpg_hashes PASSED
tests/test_sshcmd.py::test_ipv4_ipv6_rsync PASSED
tests/test_vm_alloc.py::test_ticket PASSED
tests/test_vm_alloc.py::test_ticket_with_args PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_raises PASSED
tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSED
tests/test_worker_limits.py::test_predicate_worker_limit PASSED
tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSED
tests/test_worker_limits.py::test_group_worker_limit PASSED
tests/test_worker_limits.py::test_worker_limit_info PASSED
tests/test_worker_limits.py::test_string_counter PASSED
tests/test_worker_manager.py::TestPrioQueue::test_queue_order PASSED
tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSED
tests/test_worker_manager.py::TestPrioQueue::test_push_back PASSED
tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSED
tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected FAILED
tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED
tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED
tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED
tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED
tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED
tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED
tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED
tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED
tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED
tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling PASSED
tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED
tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect FAILED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities PASSED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED
tests/run/test_copr_prune_results.py::TestPruneResults::test_run SKIPPED (Fixme or remove, test doesn't work.)
tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled SKIPPED (Fixme or remove, test doesn't work.)FAILED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities 
tests/run/test_copr_prune_results.py::TestPruneResults::test_main PASSED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED
tests/run/test_copr_prune_results.py::TestPruneResults::test_run SKIPPED (Fixme or remove, test doesn't work.)SKIPPED (Fixme or remove, test doesn't work.)
tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled 
==================================== ERRORS ====================================
___ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[True] ___
  + Exception Group Traceback (most recent call last):
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call
  |     result: Optional[TResult] = func()
  |                                 ^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in <lambda>
  |     lambda: ihook(item=item, **kwds), when=when, reraise=reraise
  |             ^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__
  |     return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec
  |     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall
  |     return outcome.get_result()
  |            ^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result
  |     raise exc.with_traceback(exc.__traceback__)
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall
  |     res = hook_impl.function(*args)
  |           ^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown
  |     item.session._setupstate.teardown_exact(nextitem)
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact
  |     raise exceptions[0]
  | ExceptionGroup: errors while tearing down <Function test_copr_repo_timeouted_check[True]> (2 sub-exceptions)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture
    |     _call_with_optional_argument(func, method)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument
    |     func(arg)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method
    |     shutil.rmtree(self.workdir)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repo1cx92qr2'
    +---------------- 2 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory
    |     shutil.rmtree(directory)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-zgin58th'
    +------------------------------------
__ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[False] ___
  + Exception Group Traceback (most recent call last):
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call
  |     result: Optional[TResult] = func()
  |                                 ^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in <lambda>
  |     lambda: ihook(item=item, **kwds), when=when, reraise=reraise
  |             ^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__
  |     return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec
  |     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall
  |     return outcome.get_result()
  |            ^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result
  |     raise exc.with_traceback(exc.__traceback__)
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall
  |     res = hook_impl.function(*args)
  |           ^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown
  |     item.session._setupstate.teardown_exact(nextitem)
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact
  |     raise exceptions[0]
  | ExceptionGroup: errors while tearing down <Function test_copr_repo_timeouted_check[False]> (2 sub-exceptions)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture
    |     _call_with_optional_argument(func, method)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument
    |     func(arg)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method
    |     shutil.rmtree(self.workdir)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repo626ctkmx'
    +---------------- 2 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory
    |     shutil.rmtree(directory)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-mgwfc7ot'
    +------------------------------------
=================================== FAILURES ===================================
__________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d4e30>
    def test_copr_modifyrepo_locks(self):
        with _lock(self.workdir) as opts:
            cmd = [modifyrepo, opts.directory, '--log-to-stdout']
            proc = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            try:
                proc.communicate(timeout=2)
                assert 0 # this shouldn't happen
            except subprocess.TimeoutExpired:
                proc.kill()
                out, err = proc.communicate()
                assert b"acquired lock" not in out
                assert b"acquired lock" not in err
    
        # with released lock
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
    
>       out, err = proc.communicate(timeout=5)
tests/test_modifyrepo.py:92: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib64/python3.12/subprocess.py:1209: in communicate
    stdout, stderr = self._communicate(input, endtime, timeout)
/usr/lib64/python3.12/subprocess.py:2109: in _communicate
    self._check_timeout(endtime, orig_timeout, stdout, stderr)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <Popen: returncode: None args: ['run/copr-repo', '/tmp/copr-test-copr-reponl...>
endtime = 1184086.099095219, orig_timeout = 5, stdout_seq = [], stderr_seq = []
skip_check_and_raise = False
    def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
                       skip_check_and_raise=False):
        """Convenience for checking if a timeout has expired."""
        if endtime is None:
            return
        if skip_check_and_raise or _time() > endtime:
>           raise TimeoutExpired(
                    self.args, orig_timeout,
                    output=b''.join(stdout_seq) if stdout_seq else None,
                    stderr=b''.join(stderr_seq) if stderr_seq else None)
E           subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-reponlewtgbd', '--log-to-stdout']' timed out after 5 seconds
/usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:215 Lock 72057592444379088 acquired on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:247 Lock 72057592444379088 released on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
_____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7e30>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04d95b0>
run_bg = True
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
>           assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
E           AssertionError: assert False
E            +  where False = call_copr_repo('/tmp/copr-backend-test-zgin58th/john/empty/fedora-rawhide-x86_64', add=['00000002-example'])
tests/test_modifyrepo.py:617: AssertionError
_____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7fb0>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04ebbf0>
run_bg = False
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
>           assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
E           AssertionError: assert False
E            +  where False = call_copr_repo('/tmp/copr-backend-test-mgwfc7ot/john/empty/fedora-rawhide-x86_64', add=['00000002-example'])
tests/test_modifyrepo.py:617: AssertionError
___________ TestLimitedWorkerManager.test_that_limits_are_respected ____________
self = <tests.test_worker_manager.TestLimitedWorkerManager object at 0xffffffa11e1bb0>
mc_time = <MagicMock name='time' id='72057592442934176'>
_mc_sleep = <MagicMock name='sleep' id='72057592443198624'>
caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffa0ee4bc0>
    @patch('copr_common.worker_manager.time.sleep')
    @patch('copr_common.worker_manager.time.time')
    def test_that_limits_are_respected(self, mc_time, _mc_sleep, caplog):
        # each time.time() call incremented by 1
        self.worker_manager.task_sleep = 5
        self.worker_manager.worker_timeout_start = 1000
        mc_time.side_effect = range(1000)
        self.worker_manager.run(timeout=150)
        messages = [
            "Task '4' skipped, limit info: 'even', "
            "matching: worker:0, worker:2",
            "Task '6' skipped, limit info: 'even', "
            "matching: worker:0, worker:2",
            "Task '7' skipped, limit info: 'odd', "
            "matching: worker:1, worker:3, worker:5",
            "Task '8' skipped, limit info: 'even', "
            "matching: worker:0, worker:2",
            "Task '9' skipped, limit info: 'odd', "
            "matching: worker:1, worker:3, worker:5",
        ]
        for msg in messages:
            assert ('root', logging.DEBUG, msg) in caplog.record_tuples
    
        # Even though the "even" limit kicked-out task 4, the task 5 is still
        # successfully started because that's the third "odd" task.  The rest of
        # tasks is just skipped.
        assert ('root', logging.INFO,
                "Starting worker worker:5, task.priority=0") in \
            caplog.record_tuples
    
        # finish the task now
        self.redis.hset("worker:5", "status", "0")
    
        self.setup_tasks()  # re-calculate limits
        self.worker_manager.run(timeout=150)
    
        # check worker manager recognized the finished task
>       assert ('root', logging.INFO, "Finished worker worker:5") \
                in caplog.record_tuples
E       assert ('root', 20, 'Finished worker worker:5') in [('root', 10, 'Worker.run() start at time 0'), ('root', 20, 'Starting worker worker:0, task.priority=0'), ('root', 10, "background pid=1113786 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, "starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker worker:1, task.priority=0'), ...]
E        +  where [('root', 10, 'Worker.run() start at time 0'), ('root', 20, 'Starting worker worker:0, task.priority=0'), ('root', 10, "background pid=1113786 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, "starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker worker:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffffa0ee4bc0>.record_tuples
tests/test_worker_manager.py:241: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 0
INFO     root:worker_manager.py:482 Starting worker worker:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113786 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113790 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:2, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113796 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:3, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113801 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:5, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113802 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113790
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113786
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113796
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113801
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113802
DEBUG    root:test_worker_manager.py:73 cleaned up 5, started 5
DEBUG    root:worker_manager.py:475 Reaped 5 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 162
DEBUG    root:worker_manager.py:383 Task 0 already has a worker process
DEBUG    root:worker_manager.py:383 Task 1 already has a worker process
DEBUG    root:worker_manager.py:383 Task 2 already has a worker process
DEBUG    root:worker_manager.py:383 Task 3 already has a worker process
DEBUG    root:worker_manager.py:383 Task 3 already has a worker process
DEBUG    root:worker_manager.py:383 Task 3 already has a worker process
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:383 Task 5 already has a worker process
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
DEBUG    root:worker_manager.py:425 Worker.run() start at time 176
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker worker:2
DEBUG    root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker worker:3
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker worker:5
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:test_worker_manager.py:73 cleaned up 0, started 0
DEBUG    root:worker_manager.py:475 Reaped 0 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 333
______________ TestWorkerManager.test_slow_priority_queue_filling ______________
self = <tests.test_worker_manager.TestWorkerManager object at 0xffffffa11e33b0>
    def test_slow_priority_queue_filling(self):
        """
        We discovered that adding tasks to a priority queue was a bottleneck
        when having a large (70k+ builds) queue, see #2095. Make sure this never
        happen again.
        """
        tasks = [ToyQueueTask(i) for i in range(100000)]
    
        # We need to run this test with logging only INFO, otherwise we waste
        # around 5 seconds just on running self.log.debug because we need to
        # connect to redis for each call
        # The point of this test is to make sure that adding tasks to priority
        # queue is not a bottleneck on production, and we don't use DEBUG there
        # anyway.
        log.setLevel(logging.INFO)
    
        t1 = time.time()
        for task in tasks:
            self.worker_manager.add_task(task)
        t2 = time.time()
    
        # It should actually be faster than 1 second but I am adding one to
        # prevent false alarms in case somebody has a slow machine
>       assert t2 - t1 < 2
E       assert (1701742664.3490565 - 1701742641.8655748) < 2
tests/test_worker_manager.py:314: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
_____________ TestActionWorkerManager.test_run_starts_the_workers ______________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e31a0>
    def test_run_starts_the_workers(self):
        self.worker_manager.run(timeout=0.0001)
        workers = self.workers()
>       assert len(workers) == 1
E       assert 0 == 1
E        +  where 0 = len([])
tests/test_worker_manager.py:356: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742665.821652
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742665.6797628:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114214 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742665.6797628:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.31961703300476074 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742665.6797628:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114214
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742682.6637995
__________ TestActionWorkerManager.test_delete_not_allocated_workers ___________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>
    def test_delete_not_allocated_workers(self):
        self.worker_manager.run(timeout=0.0001)
>       assert self.w0 in self.workers()
E       AssertionError: assert 'toy:1701742684.3004808:0' in []
E        +  where 'toy:1701742684.3004808:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>.w0
E        +  and   [] = <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>>()
E        +    where <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>.workers
tests/test_worker_manager.py:373: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742684.3496358
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742684.3004808:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114276 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742684.3004808:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.19505071640014648 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742684.3004808:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114276
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742701.2640479
___________ TestActionWorkerManager.test_delete_not_started_workers ____________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e15e0>
    def test_delete_not_started_workers(self):
        self.worker_manager.environ = {'FAIL_EARLY': '1'}
        self.worker_manager.worker_timeout_start = 0
        self.worker_manager.run(timeout=0.0001)
>       assert self.workers() == [self.w0]
E       AssertionError: assert [] == ['toy:1701742703.2191424:0']
E         Right contains one more item: 'toy:1701742703.2191424:0'
E         Full diff:
E         - ['toy:1701742703.2191424:0']
E         + []
tests/test_worker_manager.py:390: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742703.2891357
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742703.2191424:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114339 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742703.2191424:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.24625587463378906 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742703.2191424:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114339
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742718.4848697
__ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED_PID] __
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d30>
mc_time = <MagicMock name='time' id='72057592336928928'>
fail = 'FAIL_STARTED_PID'
    @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED'])
    @patch('copr_common.worker_manager.time.time')
    def test_delete_not_finished_workers(self, mc_time, fail):
        self.worker_manager.environ = {fail: '1'}
        self.worker_manager.worker_timeout_deadcheck = 0.4
    
        # each time.time() call incremented by 1
        mc_time.side_effect = range(1000)
    
        # first loop just starts the toy:0 worker
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
    
        params = self.wait_field(self.w0, 'started')
        assert self.w0 in self.workers()
        assert 'started' in params
    
        if fail == 'FAIL_STARTED':
            # make sure kernel cleans up the process, so the next wm.run()
            # certainly sets the 'delete' flag
            wait_pid_exit(params['PID'])
    
        # toy 0 is marked for deleting
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
>       assert 'delete' in self.redis.hgetall(self.w0)
E       AssertionError: assert 'delete' in {}
E        +  where {} = <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>>('toy:1701742719.6343882:0')
E        +    where <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>> = Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>.hgetall
E        +      where Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d30>.redis
E        +    and   'toy:1701742719.6343882:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d30>.w0
tests/test_worker_manager.py:428: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 0
INFO     root:worker_manager.py:482 Starting worker toy:1701742719.6343882:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114387 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742719.6343882:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742719.6343882:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114387
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 12
DEBUG    root:worker_manager.py:425 Worker.run() start at time 14
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742719.6343882:0
INFO     root:worker_manager.py:482 Starting worker toy:1701742719.6343882:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114463 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742719.6343882:1', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742719.6343882:1', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114463
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 29
____ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED] ____
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1af0>
mc_time = <MagicMock name='time' id='72057592336975584'>, fail = 'FAIL_STARTED'
    @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED'])
    @patch('copr_common.worker_manager.time.time')
    def test_delete_not_finished_workers(self, mc_time, fail):
        self.worker_manager.environ = {fail: '1'}
        self.worker_manager.worker_timeout_deadcheck = 0.4
    
        # each time.time() call incremented by 1
        mc_time.side_effect = range(1000)
    
        # first loop just starts the toy:0 worker
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
    
        params = self.wait_field(self.w0, 'started')
        assert self.w0 in self.workers()
        assert 'started' in params
    
        if fail == 'FAIL_STARTED':
            # make sure kernel cleans up the process, so the next wm.run()
            # certainly sets the 'delete' flag
            wait_pid_exit(params['PID'])
    
        # toy 0 is marked for deleting
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
>       assert 'delete' in self.redis.hgetall(self.w0)
E       AssertionError: assert 'delete' in {}
E        +  where {} = <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>>('toy:1701742759.4803448:0')
E        +    where <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>> = Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>.hgetall
E        +      where Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1af0>.redis
E        +    and   'toy:1701742759.4803448:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1af0>.w0
tests/test_worker_manager.py:428: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 0
INFO     root:worker_manager.py:482 Starting worker toy:1701742759.4803448:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114527 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742759.4803448:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742759.4803448:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114527
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 12
DEBUG    root:worker_manager.py:425 Worker.run() start at time 14
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742759.4803448:0
INFO     root:worker_manager.py:482 Starting worker toy:1701742759.4803448:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114593 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742759.4803448:1', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742759.4803448:1', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114593
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 29
___________________ TestActionWorkerManager.test_all_passed ____________________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e0560>
caplog = <_pytest.logging.LogCaptureFixture object at 0xffffff9a9dbdd0>
    def test_all_passed(self, caplog):
        # It is a lot of fun with Popen().  It seems it has some zombie reaping
        # mechanism.  If the calling function objects are destroyed (including
        # the Popen() return value reference), the future call to Popen() seems
        # to just reap the old Popen() processes.
        self.worker_manager.expected_terminations_in_cycle = 5
    
        self.worker_manager.run(timeout=100)
        for i in range(0, 10):
            smsg = "Starting worker {}{}, task.priority=0"
            assert ('root', 20, smsg.format(self.wprefix, i)) in \
                caplog.record_tuples
>           assert ('root', 20, 'Finished worker {}{}'.format(self.wprefix, i)) in caplog.record_tuples
E           assert ('root', 20, 'Finished worker toy:1701742799.3355937:0') in [('root', 10, 'Worker.run() start at time 1701742799.4097264'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1701742799.3355937:0, task.priority=0'), ('root', 10, "background pid=1114657 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742799.3355937:0', '0'])"), ('root', 10, "starting-on-background-took 0.23563551902770996 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742799.3355937:0', '0'])"), ('root', 20, 'Starting worker toy:1701742799.3355937:1, task.priority=0'), ...]
E            +  where [('root', 10, 'Worker.run() start at time 1701742799.4097264'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1701742799.3355937:0, task.priority=0'), ('root', 10, "background pid=1114657 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742799.3355937:0', '0'])"), ('root', 10, "starting-on-background-took 0.23563551902770996 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742799.3355937:0', '0'])"), ('root', 20, 'Starting worker toy:1701742799.3355937:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffff9a9dbdd0>.record_tuples
tests/test_worker_manager.py:449: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742799.4097264
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114657 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742799.3355937:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.23563551902770996 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742799.3355937:0', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114659 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742799.3355937:1', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.3319976329803467 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742799.3355937:1', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:2, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114662 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1701742799.3355937:2', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.3670384883880615 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1701742799.3355937:2', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:3, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114663 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1701742799.3355937:3', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.521808385848999 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1701742799.3355937:3', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:4, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114664 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1701742799.3355937:4', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.334399938583374 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1701742799.3355937:4', '0'])
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742799.3355937:1
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742799.3355937:4
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742799.3355937:2
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:5, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114771 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1701742799.3355937:5', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.5346870422363281 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1701742799.3355937:5', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:6, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114776 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1701742799.3355937:6', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.44614291191101074 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1701742799.3355937:6', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:7, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114780 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1701742799.3355937:7', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.509486198425293 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1701742799.3355937:7', '0'])
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742799.3355937:0
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742799.3355937:3
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:8, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114786 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1701742799.3355937:8', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.26763272285461426 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1701742799.3355937:8', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742799.3355937:9, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114789 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1701742799.3355937:9', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.30251550674438477 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1701742799.3355937:9', '0'])
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
ERROR    root:worker_manager.py:546 worker toy:1701742799.3355937:5 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742799.3355937:6 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742799.3355937:7 failed to start
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
ERROR    root:worker_manager.py:546 worker toy:1701742799.3355937:8 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742799.3355937:9 failed to start
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114771
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114776
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114780
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114786
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114789
DEBUG    root:test_worker_manager.py:73 cleaned up 5, started 10
DEBUG    root:worker_manager.py:475 Reaped 5 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742875.25648
_________ TestActionWorkerManager.test_empty_queue_but_workers_running _________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e13a0>
    def test_empty_queue_but_workers_running(self):
        'check that sleep(1) is done if queue is empty, but some workers exist'
    
        self.worker_manager.clean_tasks()
    
        # only one task, but it will take some time.
        self.worker_manager.task_sleep = 0.5
        self.worker_manager.add_task(ToyQueueTask(0))
    
        # start the worker
        self.worker_manager.run(timeout=0.0001) # start them task
    
        with patch('copr_common.worker_manager.time.sleep') as sleep:
            # we can spawn more workers, but queue is empty
            self.worker_manager.run(timeout=0.0001)
            assert sleep.called
>       assert len(self.worker_manager.worker_ids()) == 1
E       assert 0 == 1
E        +  where 0 = len([])
E        +    where [] = <bound method WorkerManager.worker_ids of <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9f79e0>>()
E        +      where <bound method WorkerManager.worker_ids of <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9f79e0>> = <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9f79e0>.worker_ids
E        +        where <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9f79e0> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e13a0>.worker_manager
tests/test_worker_manager.py:477: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742901.6439526
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742901.559636:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1115000 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742901.559636:0', '0.5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.16889166831970215 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742901.559636:0', '0.5'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1115000
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742919.3888428
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742919.4243865
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:test_worker_manager.py:73 cleaned up 0, started 0
DEBUG    root:worker_manager.py:475 Reaped 0 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742919.4618316
_____________ TestActionWorkerManager.test_that_we_check_aliveness _____________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e38c0>
    def test_that_we_check_aliveness(self):
        """
        Worker Manager checks whether worker is running each 'worker_timeout_deadcheck'
        period, check whether it works
        """
        self.worker_manager.task_sleep = 3 # assure task takes some time
        self.worker_manager.clean_tasks()
        self.worker_manager.add_task(ToyQueueTask(0))
        self.worker_manager.worker_timeout_start = 1
        self.worker_manager.worker_timeout_deadcheck = 1.5
    
        # start the worker
        self.worker_manager.run(timeout=0.0001)
    
        # let the task start
        self.wait_field(self.w0, 'PID')
    
        # timeout for liveness check not yet left
        self.worker_manager.run(timeout=0.0001)
        params = self.redis.hgetall(self.w0)
        if 'checked' in params:
            # slow builder, the delay between previous two run() calls were so
            # long so the second one managed to even check whether the worker is
            # alive.  So if that happened, the delay needs to be larger than
            # deadcheck at least.
            checked = float(params['checked'])
            started = float(params['allocated'])
            assert started + self.worker_manager.worker_timeout_deadcheck <= checked
    
        # time for check..
        time.sleep(1.5)
        self.worker_manager.run(timeout=0.0001)
        params = self.redis.hgetall(self.w0)
>       assert 'checked' in params
E       AssertionError: assert 'checked' in {}
tests/test_worker_manager.py:522: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742921.4213786
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742921.3437912:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1115067 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742921.3437912:0', '3'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.23810648918151855 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742921.3437912:0', '3'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1115067
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742936.969776
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742942.631004
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742921.3437912:0
DEBUG    root:test_worker_manager.py:73 cleaned up 0, started 0
DEBUG    root:worker_manager.py:475 Reaped 0 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742942.691302
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742944.2038224
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:test_worker_manager.py:73 cleaned up 0, started 0
DEBUG    root:worker_manager.py:475 Reaped 0 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742944.2555
_____________ TestActionWorkerManager.test_max_workers_has_effect ______________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>
    def test_max_workers_has_effect(self):
        self.worker_manager.max_workers = 1
        self.worker_manager.run(timeout=1)
>       assert self.w0 in self.workers()
E       AssertionError: assert 'toy:1701742945.6674538:0' in []
E        +  where 'toy:1701742945.6674538:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>.w0
E        +  and   [] = <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>>()
E        +    where <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>.workers
tests/test_worker_manager.py:533: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742945.740898
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742945.6674538:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1115145 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742945.6674538:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.18668007850646973 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742945.6674538:0', '0'])
DEBUG    root:worker_manager.py:443 Worker count on a limit 1
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1115145
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742959.2498395
=============================== warnings summary ===============================
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API
    warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning)
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`.
  Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
    declare_namespace(pkg)
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
  /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    override_loader = get_loader(override_package_name)
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
  /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    return find_loader(fullname)
tests/test_frontend.py:14
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_frontend.py:29
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:18
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:24
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:21
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:26
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:31
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:36
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_action.py: 5 warnings
tests/test_modifyrepo.py: 16 warnings
tests/run/test_copr_prune_results.py: 3 warnings
  /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior.
    warnings.warn(
tests/test_action.py::TestAction::test_comps_create
  /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
    now = datetime.utcnow()
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker
  /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14.
    rv = reductor(4)
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True]
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False]
FAILED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected
FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID]
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED]
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect
ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True]
ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False]
= 14 failed, 162 passed, 11 skipped, 47 warnings, 2 errors in 2513.55s (0:41:53) =
SKIPPED (Fixme or remove, test doesn't work.)[INFO/MainProcess] process shutting down
[INFO/MainProcess] process shutting down
tests/run/test_copr_prune_results.py::TestPruneResults::test_main SKIPPED (Fixme or remove, test doesn't work.)
==================================== ERRORS ====================================
__ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[False] ___
  + Exception Group Traceback (most recent call last):
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call
  |     result: Optional[TResult] = func()
  |                                 ^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in <lambda>
  |     lambda: ihook(item=item, **kwds), when=when, reraise=reraise
  |             ^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__
  |     return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec
  |     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall
  |     return outcome.get_result()
  |            ^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result
  |     raise exc.with_traceback(exc.__traceback__)
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall
  |     res = hook_impl.function(*args)
  |           ^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown
  |     item.session._setupstate.teardown_exact(nextitem)
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact
  |     raise exceptions[0]
  | ExceptionGroup: errors while tearing down <Function test_copr_repo_timeouted_check[False]> (2 sub-exceptions)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture
    |     _call_with_optional_argument(func, method)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument
    |     func(arg)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method
    |     shutil.rmtree(self.workdir)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repok3h7000w'
    +---------------- 2 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory
    |     shutil.rmtree(directory)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-710e3qgv'
    +------------------------------------
=================================== FAILURES ===================================
__________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d4e30>
    def test_copr_modifyrepo_locks(self):
        with _lock(self.workdir) as opts:
            cmd = [modifyrepo, opts.directory, '--log-to-stdout']
            proc = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            try:
                proc.communicate(timeout=2)
                assert 0 # this shouldn't happen
            except subprocess.TimeoutExpired:
                proc.kill()
                out, err = proc.communicate()
                assert b"acquired lock" not in out
                assert b"acquired lock" not in err
    
        # with released lock
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
    
>       out, err = proc.communicate(timeout=5)
tests/test_modifyrepo.py:92: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib64/python3.12/subprocess.py:1209: in communicate
    stdout, stderr = self._communicate(input, endtime, timeout)
/usr/lib64/python3.12/subprocess.py:2109: in _communicate
    self._check_timeout(endtime, orig_timeout, stdout, stderr)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <Popen: returncode: None args: ['run/copr-repo', '/tmp/copr-test-copr-reponl...>
endtime = 1184086.099095219, orig_timeout = 5, stdout_seq = [], stderr_seq = []
skip_check_and_raise = False
    def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
                       skip_check_and_raise=False):
        """Convenience for checking if a timeout has expired."""
        if endtime is None:
            return
        if skip_check_and_raise or _time() > endtime:
>           raise TimeoutExpired(
                    self.args, orig_timeout,
                    output=b''.join(stdout_seq) if stdout_seq else None,
                    stderr=b''.join(stderr_seq) if stderr_seq else None)
E           subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-reponlewtgbd', '--log-to-stdout']' timed out after 5 seconds
/usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:215 Lock 72057592444379088 acquired on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:247 Lock 72057592444379088 released on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
_____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7e30>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04d95b0>
run_bg = True
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
            assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
            # sys.exit() can not be used in testsuite
            os._exit(0)  # pylint: disable=protected-access
    
        with _lock(chrootdir):
            # give the child some time to fill its Redis keys
            sleeper = 1
            while True:
                if len(self.redis.keys()) > 0:
                    break
                sleeper += 1
                time.sleep(0.1)
>               assert sleeper < 10*15  # 15s
E               assert 150 < (10 * 15)
tests/test_modifyrepo.py:629: AssertionError
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592432469600 on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:215 Lock 72057592432469600 acquired on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592432469600 on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:247 Lock 72057592432469600 released on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
_____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7fb0>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04eb560>
run_bg = False
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
>           assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
E           AssertionError: assert False
E            +  where False = call_copr_repo('/tmp/copr-backend-test-710e3qgv/john/empty/fedora-rawhide-x86_64', add=['00000002-example'])
tests/test_modifyrepo.py:617: AssertionError
______________ TestWorkerManager.test_slow_priority_queue_filling ______________
self = <tests.test_worker_manager.TestWorkerManager object at 0xffffffa11e33b0>
    def test_slow_priority_queue_filling(self):
        """
        We discovered that adding tasks to a priority queue was a bottleneck
        when having a large (70k+ builds) queue, see #2095. Make sure this never
        happen again.
        """
        tasks = [ToyQueueTask(i) for i in range(100000)]
    
        # We need to run this test with logging only INFO, otherwise we waste
        # around 5 seconds just on running self.log.debug because we need to
        # connect to redis for each call
        # The point of this test is to make sure that adding tasks to priority
        # queue is not a bottleneck on production, and we don't use DEBUG there
        # anyway.
        log.setLevel(logging.INFO)
    
        t1 = time.time()
        for task in tasks:
            self.worker_manager.add_task(task)
        t2 = time.time()
    
        # It should actually be faster than 1 second but I am adding one to
        # prevent false alarms in case somebody has a slow machine
>       assert t2 - t1 < 2
E       assert (1701742675.4041028 - 1701742653.6147635) < 2
tests/test_worker_manager.py:314: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
_____________ TestActionWorkerManager.test_run_starts_the_workers ______________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e31a0>
    def test_run_starts_the_workers(self):
        self.worker_manager.run(timeout=0.0001)
        workers = self.workers()
>       assert len(workers) == 1
E       assert 0 == 1
E        +  where 0 = len([])
tests/test_worker_manager.py:356: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742675.9483504
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742675.9230762:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114250 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742675.9230762:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.14367270469665527 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742675.9230762:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114250
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742691.6522806
__________ TestActionWorkerManager.test_delete_not_allocated_workers ___________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>
    def test_delete_not_allocated_workers(self):
        self.worker_manager.run(timeout=0.0001)
>       assert self.w0 in self.workers()
E       AssertionError: assert 'toy:1701742693.193294:0' in []
E        +  where 'toy:1701742693.193294:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>.w0
E        +  and   [] = <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>>()
E        +    where <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e2b10>.workers
tests/test_worker_manager.py:373: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742693.2687619
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742693.193294:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114308 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742693.193294:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.19427752494812012 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742693.193294:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114308
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742710.0384638
___________ TestActionWorkerManager.test_delete_not_started_workers ____________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e15e0>
    def test_delete_not_started_workers(self):
        self.worker_manager.environ = {'FAIL_EARLY': '1'}
        self.worker_manager.worker_timeout_start = 0
        self.worker_manager.run(timeout=0.0001)
>       assert self.workers() == [self.w0]
E       AssertionError: assert [] == ['toy:1701742711.3906388:0']
E         Right contains one more item: 'toy:1701742711.3906388:0'
E         Full diff:
E         - ['toy:1701742711.3906388:0']
E         + []
tests/test_worker_manager.py:390: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742711.4502523
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742711.3906388:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114356 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742711.3906388:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.17980194091796875 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742711.3906388:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114356
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742726.4727347
__ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED_PID] __
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d30>
mc_time = <MagicMock name='time' id='72057592336928064'>
fail = 'FAIL_STARTED_PID'
    @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED'])
    @patch('copr_common.worker_manager.time.time')
    def test_delete_not_finished_workers(self, mc_time, fail):
        self.worker_manager.environ = {fail: '1'}
        self.worker_manager.worker_timeout_deadcheck = 0.4
    
        # each time.time() call incremented by 1
        mc_time.side_effect = range(1000)
    
        # first loop just starts the toy:0 worker
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
    
        params = self.wait_field(self.w0, 'started')
        assert self.w0 in self.workers()
        assert 'started' in params
    
        if fail == 'FAIL_STARTED':
            # make sure kernel cleans up the process, so the next wm.run()
            # certainly sets the 'delete' flag
            wait_pid_exit(params['PID'])
    
        # toy 0 is marked for deleting
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
>       assert 'delete' in self.redis.hgetall(self.w0)
E       AssertionError: assert 'delete' in {}
E        +  where {} = <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>>('toy:1701742727.3382134:0')
E        +    where <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>> = Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>.hgetall
E        +      where Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d30>.redis
E        +    and   'toy:1701742727.3382134:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d30>.w0
tests/test_worker_manager.py:428: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 0
INFO     root:worker_manager.py:482 Starting worker toy:1701742727.3382134:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114417 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742727.3382134:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742727.3382134:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114417
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 12
DEBUG    root:worker_manager.py:425 Worker.run() start at time 14
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:553 checking worker toy:1701742727.3382134:0
ERROR    root:worker_manager.py:557 dead worker toy:1701742727.3382134:0
INFO     root:worker_manager.py:482 Starting worker toy:1701742727.3382134:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114490 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742727.3382134:1', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742727.3382134:1', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114490
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 30
____ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED] ____
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1af0>
mc_time = <MagicMock name='time' id='72057592337024352'>, fail = 'FAIL_STARTED'
    @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED'])
    @patch('copr_common.worker_manager.time.time')
    def test_delete_not_finished_workers(self, mc_time, fail):
        self.worker_manager.environ = {fail: '1'}
        self.worker_manager.worker_timeout_deadcheck = 0.4
    
        # each time.time() call incremented by 1
        mc_time.side_effect = range(1000)
    
        # first loop just starts the toy:0 worker
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
    
        params = self.wait_field(self.w0, 'started')
        assert self.w0 in self.workers()
        assert 'started' in params
    
        if fail == 'FAIL_STARTED':
            # make sure kernel cleans up the process, so the next wm.run()
            # certainly sets the 'delete' flag
            wait_pid_exit(params['PID'])
    
        # toy 0 is marked for deleting
        with patch('copr_common.worker_manager.time.sleep'):
            self.worker_manager.run(timeout=1)
>       assert 'delete' in self.redis.hgetall(self.w0)
E       AssertionError: assert 'delete' in {}
E        +  where {} = <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>>('toy:1701742766.0881984:0')
E        +    where <bound method HashCommands.hgetall of Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>> = Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>>.hgetall
E        +      where Redis<ConnectionPool<Connection<host=localhost,port=7777,db=9>>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1af0>.redis
E        +    and   'toy:1701742766.0881984:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1af0>.w0
tests/test_worker_manager.py:428: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 0
INFO     root:worker_manager.py:482 Starting worker toy:1701742766.0881984:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114547 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742766.0881984:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742766.0881984:0', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114547
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 12
DEBUG    root:worker_manager.py:425 Worker.run() start at time 14
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:553 checking worker toy:1701742766.0881984:0
ERROR    root:worker_manager.py:557 dead worker toy:1701742766.0881984:0
INFO     root:worker_manager.py:482 Starting worker toy:1701742766.0881984:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114627 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742766.0881984:1', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742766.0881984:1', '0'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114627
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 30
___________________ TestActionWorkerManager.test_all_passed ____________________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e0560>
caplog = <_pytest.logging.LogCaptureFixture object at 0xffffff9a9e7bf0>
    def test_all_passed(self, caplog):
        # It is a lot of fun with Popen().  It seems it has some zombie reaping
        # mechanism.  If the calling function objects are destroyed (including
        # the Popen() return value reference), the future call to Popen() seems
        # to just reap the old Popen() processes.
        self.worker_manager.expected_terminations_in_cycle = 5
    
        self.worker_manager.run(timeout=100)
        for i in range(0, 10):
            smsg = "Starting worker {}{}, task.priority=0"
            assert ('root', 20, smsg.format(self.wprefix, i)) in \
                caplog.record_tuples
>           assert ('root', 20, 'Finished worker {}{}'.format(self.wprefix, i)) in caplog.record_tuples
E           assert ('root', 20, 'Finished worker toy:1701742810.3628993:0') in [('root', 10, 'Worker.run() start at time 1701742810.545179'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1701742810.3628993:0, task.priority=0'), ('root', 10, "background pid=1114699 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742810.3628993:0', '0'])"), ('root', 10, "starting-on-background-took 0.36362433433532715 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742810.3628993:0', '0'])"), ('root', 20, 'Starting worker toy:1701742810.3628993:1, task.priority=0'), ...]
E            +  where [('root', 10, 'Worker.run() start at time 1701742810.545179'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1701742810.3628993:0, task.priority=0'), ('root', 10, "background pid=1114699 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742810.3628993:0', '0'])"), ('root', 10, "starting-on-background-took 0.36362433433532715 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742810.3628993:0', '0'])"), ('root', 20, 'Starting worker toy:1701742810.3628993:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffff9a9e7bf0>.record_tuples
tests/test_worker_manager.py:449: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742810.545179
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114699 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742810.3628993:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.36362433433532715 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742810.3628993:0', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114700 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742810.3628993:1', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.3700237274169922 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1701742810.3628993:1', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:2, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114703 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1701742810.3628993:2', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 1.8651106357574463 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1701742810.3628993:2', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:3, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114704 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1701742810.3628993:3', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.36281919479370117 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1701742810.3628993:3', '0'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:4, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114708 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1701742810.3628993:4', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.16562676429748535 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1701742810.3628993:4', '0'])
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:2 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:0 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:1 failed to start
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:5, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114810 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1701742810.3628993:5', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.39406347274780273 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1701742810.3628993:5', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:6, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114815 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1701742810.3628993:6', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.36484646797180176 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1701742810.3628993:6', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:7, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114816 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1701742810.3628993:7', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.9295260906219482 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1701742810.3628993:7', '0'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:4 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:3 failed to start
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:8, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114819 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1701742810.3628993:8', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.3607137203216553 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1701742810.3628993:8', '0'])
INFO     root:worker_manager.py:482 Starting worker toy:1701742810.3628993:9, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1114820 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1701742810.3628993:9', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.3947150707244873 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1701742810.3628993:9', '0'])
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742810.3628993:4
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742810.3628993:3
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742810.3628993:0
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742810.3628993:1
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742810.3628993:2
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:443 Worker count on a limit 5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:6 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:5 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:7 failed to start
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:9 failed to start
ERROR    root:worker_manager.py:546 worker toy:1701742810.3628993:8 failed to start
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114699
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114700
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114703
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114704
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114708
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114819
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114815
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114810
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114816
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1114820
DEBUG    root:test_worker_manager.py:73 cleaned up 10, started 10
DEBUG    root:worker_manager.py:475 Reaped 10 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742887.2700427
_________ TestActionWorkerManager.test_empty_queue_but_workers_running _________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e13a0>
    def test_empty_queue_but_workers_running(self):
        'check that sleep(1) is done if queue is empty, but some workers exist'
    
        self.worker_manager.clean_tasks()
    
        # only one task, but it will take some time.
        self.worker_manager.task_sleep = 0.5
        self.worker_manager.add_task(ToyQueueTask(0))
    
        # start the worker
        self.worker_manager.run(timeout=0.0001) # start them task
    
        with patch('copr_common.worker_manager.time.sleep') as sleep:
            # we can spawn more workers, but queue is empty
            self.worker_manager.run(timeout=0.0001)
            assert sleep.called
>       assert len(self.worker_manager.worker_ids()) == 1
E       assert 0 == 1
E        +  where 0 = len([])
E        +    where [] = <bound method WorkerManager.worker_ids of <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9e5610>>()
E        +      where <bound method WorkerManager.worker_ids of <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9e5610>> = <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9e5610>.worker_ids
E        +        where <tests.test_worker_manager.ToyActionWorkerManager object at 0xffffff9a9e5610> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e13a0>.worker_manager
tests/test_worker_manager.py:477: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742906.8798916
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742906.7942097:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1115022 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742906.7942097:0', '0.5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.22150254249572754 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742906.7942097:0', '0.5'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1115022
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742924.0957918
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742924.197855
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:test_worker_manager.py:73 cleaned up 0, started 0
DEBUG    root:worker_manager.py:475 Reaped 0 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742924.3158844
_____________ TestActionWorkerManager.test_that_we_check_aliveness _____________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e38c0>
    def test_that_we_check_aliveness(self):
        """
        Worker Manager checks whether worker is running each 'worker_timeout_deadcheck'
        period, check whether it works
        """
        self.worker_manager.task_sleep = 3 # assure task takes some time
        self.worker_manager.clean_tasks()
        self.worker_manager.add_task(ToyQueueTask(0))
        self.worker_manager.worker_timeout_start = 1
        self.worker_manager.worker_timeout_deadcheck = 1.5
    
        # start the worker
        self.worker_manager.run(timeout=0.0001)
    
        # let the task start
        self.wait_field(self.w0, 'PID')
    
        # timeout for liveness check not yet left
        self.worker_manager.run(timeout=0.0001)
        params = self.redis.hgetall(self.w0)
        if 'checked' in params:
            # slow builder, the delay between previous two run() calls were so
            # long so the second one managed to even check whether the worker is
            # alive.  So if that happened, the delay needs to be larger than
            # deadcheck at least.
            checked = float(params['checked'])
            started = float(params['allocated'])
            assert started + self.worker_manager.worker_timeout_deadcheck <= checked
    
        # time for check..
        time.sleep(1.5)
        self.worker_manager.run(timeout=0.0001)
        params = self.redis.hgetall(self.w0)
>       assert 'checked' in params
E       AssertionError: assert 'checked' in {}
tests/test_worker_manager.py:522: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742925.3604248
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742925.3160052:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1115080 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742925.3160052:0', '3'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.26433300971984863 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742925.3160052:0', '3'])
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1115080
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742941.382401
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742946.4813821
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1701742925.3160052:0
DEBUG    root:test_worker_manager.py:73 cleaned up 0, started 0
DEBUG    root:worker_manager.py:475 Reaped 0 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742946.5279343
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742948.0411541
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:test_worker_manager.py:73 cleaned up 0, started 0
DEBUG    root:worker_manager.py:475 Reaped 0 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742948.0644407
_____________ TestActionWorkerManager.test_max_workers_has_effect ______________
self = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>
    def test_max_workers_has_effect(self):
        self.worker_manager.max_workers = 1
        self.worker_manager.run(timeout=1)
>       assert self.w0 in self.workers()
E       AssertionError: assert 'toy:1701742949.1279895:0' in []
E        +  where 'toy:1701742949.1279895:0' = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>.w0
E        +  and   [] = <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>>()
E        +    where <bound method BaseTestWorkerManager.workers of <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>> = <tests.test_worker_manager.TestActionWorkerManager object at 0xffffffa11e1d00>.workers
tests/test_worker_manager.py:533: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 1701742949.1956806
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker toy:1701742949.1279895:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1115160 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742949.1279895:0', '0'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 0.19592571258544922 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1701742949.1279895:0', '0'])
DEBUG    root:worker_manager.py:443 Worker count on a limit 1
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1115160
DEBUG    root:test_worker_manager.py:73 cleaned up 1, started 1
DEBUG    root:worker_manager.py:475 Reaped 1 processes
DEBUG    root:worker_manager.py:476 Worker.run() stop at time 1701742964.6588933
=============================== warnings summary ===============================
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API
    warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning)
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`.
  Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
    declare_namespace(pkg)
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
  /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    override_loader = get_loader(override_package_name)
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
  /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    return find_loader(fullname)
tests/test_frontend.py:14
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_frontend.py:29
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:18
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:24
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:21
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:26
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:31
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:36
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_action.py: 5 warnings
tests/test_modifyrepo.py: 16 warnings
tests/run/test_copr_prune_results.py: 3 warnings
  /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior.
    warnings.warn(
tests/test_action.py::TestAction::test_comps_create
  /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
    now = datetime.utcnow()
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker
  /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14.
    rv = reductor(4)
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True]
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False]
FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID]
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED]
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness
FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect
ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False]
= 13 failed, 163 passed, 11 skipped, 47 warnings, 1 error in 2520.85s (0:42:00) =
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process shutting down
FAILED
tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED
tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED
tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED
tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED
tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED
tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect PASSED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities PASSED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED
tests/run/test_copr_prune_results.py::TestPruneResults::test_run SKIPPED (Fixme or remove, test doesn't work.)
tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled SKIPPED (Fixme or remove, test doesn't work.)
tests/run/test_copr_prune_results.py::TestPruneResults::test_main SKIPPED (Fixme or remove, test doesn't work.)
==================================== ERRORS ====================================
___ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[True] ___
  + Exception Group Traceback (most recent call last):
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call
  |     result: Optional[TResult] = func()
  |                                 ^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in <lambda>
  |     lambda: ihook(item=item, **kwds), when=when, reraise=reraise
  |             ^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__
  |     return self._hookexec(self.name, self._hookimpls, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec
  |     return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall
  |     return outcome.get_result()
  |            ^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result
  |     raise exc.with_traceback(exc.__traceback__)
  |   File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall
  |     res = hook_impl.function(*args)
  |           ^^^^^^^^^^^^^^^^^^^^^^^^^
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown
  |     item.session._setupstate.teardown_exact(nextitem)
  |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact
  |     raise exceptions[0]
  | ExceptionGroup: errors while tearing down <Function test_copr_repo_timeouted_check[True]> (2 sub-exceptions)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture
    |     _call_with_optional_argument(func, method)
    |   File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument
    |     func(arg)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method
    |     shutil.rmtree(self.workdir)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repo1cx92qr2'
    +---------------- 2 ----------------
    | Traceback (most recent call last):
    |   File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact
    |     fin()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in <lambda>
    |     subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest))
    |                                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish
    |     raise exc
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish
    |     func()
    |   File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture
    |     next(it)
    |   File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory
    |     shutil.rmtree(directory)
    |   File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree
    |     onexc(os.lstat, path, err)
    |   File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree
    |     orig_st = os.lstat(path, dir_fd=dir_fd)
    |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-zgin58th'
    +------------------------------------
=================================== FAILURES ===================================
__________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d4e30>
    def test_copr_modifyrepo_locks(self):
        with _lock(self.workdir) as opts:
            cmd = [modifyrepo, opts.directory, '--log-to-stdout']
            proc = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            try:
                proc.communicate(timeout=2)
                assert 0 # this shouldn't happen
            except subprocess.TimeoutExpired:
                proc.kill()
                out, err = proc.communicate()
                assert b"acquired lock" not in out
                assert b"acquired lock" not in err
    
        # with released lock
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
    
>       out, err = proc.communicate(timeout=5)
tests/test_modifyrepo.py:92: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib64/python3.12/subprocess.py:1209: in communicate
    stdout, stderr = self._communicate(input, endtime, timeout)
/usr/lib64/python3.12/subprocess.py:2109: in _communicate
    self._check_timeout(endtime, orig_timeout, stdout, stderr)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <Popen: returncode: None args: ['run/copr-repo', '/tmp/copr-test-copr-reponl...>
endtime = 1184086.099095219, orig_timeout = 5, stdout_seq = [], stderr_seq = []
skip_check_and_raise = False
    def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
                       skip_check_and_raise=False):
        """Convenience for checking if a timeout has expired."""
        if endtime is None:
            return
        if skip_check_and_raise or _time() > endtime:
>           raise TimeoutExpired(
                    self.args, orig_timeout,
                    output=b''.join(stdout_seq) if stdout_seq else None,
                    stderr=b''.join(stderr_seq) if stderr_seq else None)
E           subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-reponlewtgbd', '--log-to-stdout']' timed out after 5 seconds
/usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:215 Lock 72057592444379088 acquired on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:247 Lock 72057592444379088 released on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
_____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7e30>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04d95b0>
run_bg = True
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
>           assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
E           AssertionError: assert False
E            +  where False = call_copr_repo('/tmp/copr-backend-test-zgin58th/john/empty/fedora-rawhide-x86_64', add=['00000002-example'])
tests/test_modifyrepo.py:617: AssertionError
_____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7fb0>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04ebbf0>
run_bg = False
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
            assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
            # sys.exit() can not be used in testsuite
            os._exit(0)  # pylint: disable=protected-access
    
        with _lock(chrootdir):
            # give the child some time to fill its Redis keys
            sleeper = 1
            while True:
                if len(self.redis.keys()) > 0:
                    break
                sleeper += 1
                time.sleep(0.1)
>               assert sleeper < 10*15  # 15s
E               assert 150 < (10 * 15)
tests/test_modifyrepo.py:629: AssertionError
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592432461920 on /tmp/copr-test-copr-repo626ctkmx/_@_tmp_@_copr-backend-test-mgwfc7ot_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:215 Lock 72057592432461920 acquired on /tmp/copr-test-copr-repo626ctkmx/_@_tmp_@_copr-backend-test-mgwfc7ot_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592432461920 on /tmp/copr-test-copr-repo626ctkmx/_@_tmp_@_copr-backend-test-mgwfc7ot_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:247 Lock 72057592432461920 released on /tmp/copr-test-copr-repo626ctkmx/_@_tmp_@_copr-backend-test-mgwfc7ot_@_john_@_empty_@_fedora-rawhide-x86_64.lock
___________ TestLimitedWorkerManager.test_that_limits_are_respected ____________
self = <tests.test_worker_manager.TestLimitedWorkerManager object at 0xffffffa11e1bb0>
mc_time = <MagicMock name='time' id='72057592432779744'>
_mc_sleep = <MagicMock name='sleep' id='72057592443202128'>
caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffa0534c20>
    @patch('copr_common.worker_manager.time.sleep')
    @patch('copr_common.worker_manager.time.time')
    def test_that_limits_are_respected(self, mc_time, _mc_sleep, caplog):
        # each time.time() call incremented by 1
        self.worker_manager.task_sleep = 5
        self.worker_manager.worker_timeout_start = 1000
        mc_time.side_effect = range(1000)
>       self.worker_manager.run(timeout=150)
tests/test_worker_manager.py:211: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_worker_manager.py:83: in run
    return super().run(*args, **kwargs)
/usr/lib/python3.12/site-packages/copr_common/worker_manager.py:475: in run
    self.log.debug("Reaped %s processes", self._clean_daemon_processes())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <tests.test_worker_manager.ToyWorkerManager object at 0xffffffa0535490>
    def _clean_daemon_processes(self):
        """
        Check that we are not leaving any zombies behind us
        """
        waited = super()._clean_daemon_processes()
        self.log.debug("cleaned up %s, started %s", waited, self.started_in_cycle)
        if waited != self.started_in_cycle:
            if self.expected_terminations_in_cycle is not None:
                assert self.expected_terminations_in_cycle == waited
                return waited
>           assert False
E           assert False
tests/test_worker_manager.py:78: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 0
INFO     root:worker_manager.py:482 Starting worker worker:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113727 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113729 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:2, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113731 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:3, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113735 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:5, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113742 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113727
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113729
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113735
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113742
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113731
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113635
DEBUG    root:test_worker_manager.py:73 cleaned up 6, started 5
______________ TestWorkerManager.test_slow_priority_queue_filling ______________
self = <tests.test_worker_manager.TestWorkerManager object at 0xffffffa11e33b0>
    def test_slow_priority_queue_filling(self):
        """
        We discovered that adding tasks to a priority queue was a bottleneck
        when having a large (70k+ builds) queue, see #2095. Make sure this never
        happen again.
        """
        tasks = [ToyQueueTask(i) for i in range(100000)]
    
        # We need to run this test with logging only INFO, otherwise we waste
        # around 5 seconds just on running self.log.debug because we need to
        # connect to redis for each call
        # The point of this test is to make sure that adding tasks to priority
        # queue is not a bottleneck on production, and we don't use DEBUG there
        # anyway.
        log.setLevel(logging.INFO)
    
        t1 = time.time()
        for task in tasks:
            self.worker_manager.add_task(task)
        t2 = time.time()
    
        # It should actually be faster than 1 second but I am adding one to
        # prevent false alarms in case somebody has a slow machine
>       assert t2 - t1 < 2
E       assert (1701743040.2472248 - 1701743022.4226074) < 2
tests/test_worker_manager.py:314: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
=============================== warnings summary ===============================
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API
    warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning)
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`.
  Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
    declare_namespace(pkg)
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
  /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    override_loader = get_loader(override_package_name)
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
  /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    return find_loader(fullname)
tests/test_frontend.py:14
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_frontend.py:29
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:18
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:24
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:21
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:26
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:31
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:36
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_action.py: 5 warnings
tests/test_modifyrepo.py: 16 warnings
tests/run/test_copr_prune_results.py: 3 warnings
  /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior.
    warnings.warn(
tests/test_action.py::TestAction::test_comps_create
  /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
    now = datetime.utcnow()
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker
  /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14.
    rv = reductor(4)
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True]
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False]
FAILED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected
FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling
ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True]
= 5 failed, 171 passed, 11 skipped, 47 warnings, 1 error in 2905.36s (0:48:25) =
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process shutting down
FAILED
tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED
tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED
tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED
tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED
tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED
tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED
tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness PASSED
tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect PASSED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities PASSED
tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED
tests/run/test_copr_prune_results.py::TestPruneResults::test_run SKIPPED (Fixme or remove, test doesn't work.)
tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled SKIPPED (Fixme or remove, test doesn't work.)
tests/run/test_copr_prune_results.py::TestPruneResults::test_main SKIPPED (Fixme or remove, test doesn't work.)
=================================== FAILURES ===================================
__________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d4e30>
    def test_copr_modifyrepo_locks(self):
        with _lock(self.workdir) as opts:
            cmd = [modifyrepo, opts.directory, '--log-to-stdout']
            proc = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            try:
                proc.communicate(timeout=2)
                assert 0 # this shouldn't happen
            except subprocess.TimeoutExpired:
                proc.kill()
                out, err = proc.communicate()
                assert b"acquired lock" not in out
                assert b"acquired lock" not in err
    
        # with released lock
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
    
>       out, err = proc.communicate(timeout=5)
tests/test_modifyrepo.py:92: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib64/python3.12/subprocess.py:1209: in communicate
    stdout, stderr = self._communicate(input, endtime, timeout)
/usr/lib64/python3.12/subprocess.py:2109: in _communicate
    self._check_timeout(endtime, orig_timeout, stdout, stderr)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <Popen: returncode: None args: ['run/copr-repo', '/tmp/copr-test-copr-reponl...>
endtime = 1184086.099095219, orig_timeout = 5, stdout_seq = [], stderr_seq = []
skip_check_and_raise = False
    def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq,
                       skip_check_and_raise=False):
        """Convenience for checking if a timeout has expired."""
        if endtime is None:
            return
        if skip_check_and_raise or _time() > endtime:
>           raise TimeoutExpired(
                    self.args, orig_timeout,
                    output=b''.join(stdout_seq) if stdout_seq else None,
                    stderr=b''.join(stderr_seq) if stderr_seq else None)
E           subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-reponlewtgbd', '--log-to-stdout']' timed out after 5 seconds
/usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:215 Lock 72057592444379088 acquired on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592444379088 on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
DEBUG    filelock:_api.py:247 Lock 72057592444379088 released on /tmp/copr-test-copr-reponlewtgbd/_@_tmp_@_copr-test-copr-reponlewtgbd.lock
_____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7e30>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04d95b0>
run_bg = True
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
            assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
            # sys.exit() can not be used in testsuite
            os._exit(0)  # pylint: disable=protected-access
    
        with _lock(chrootdir):
            # give the child some time to fill its Redis keys
            sleeper = 1
            while True:
                if len(self.redis.keys()) > 0:
                    break
                sleeper += 1
                time.sleep(0.1)
>               assert sleeper < 10*15  # 15s
E               assert 150 < (10 * 15)
tests/test_modifyrepo.py:629: AssertionError
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592432469600 on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:215 Lock 72057592432469600 acquired on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592432469600 on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:247 Lock 72057592432469600 released on /tmp/copr-test-copr-repo1cx92qr2/_@_tmp_@_copr-backend-test-zgin58th_@_john_@_empty_@_fedora-rawhide-x86_64.lock
_____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________
self = <tests.test_modifyrepo.TestModifyRepo object at 0xffffffa11d7fb0>
f_second_build = <tests.conftest.CoprTestFixtureContext object at 0xffffffa04eb560>
run_bg = False
    @pytest.mark.parametrize('run_bg', [True, False])
    @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'})
    def test_copr_repo_timeouted_check(self, f_second_build, run_bg):
        _unused = self
        ctx = f_second_build
        chroot = ctx.chroots[0]
        chrootdir = os.path.join(ctx.empty_dir, chroot)
        repodata = os.path.join(chrootdir, 'repodata')
    
        # empty repodata at the beginning
        empty_repodata = load_primary_xml(repodata)
        assert empty_repodata['names'] == set()
    
        pid = os.fork()
        if not pid:
            # give parent some time to lock the repo
            time.sleep(1)
            # Run the blocked (by parent) createrepo, it must finish soon
            # anway because parent will claim the task is done.
            assert call_copr_repo(chrootdir, add=[ctx.builds[1]])
            # sys.exit() can not be used in testsuite
            os._exit(0)  # pylint: disable=protected-access
    
        with _lock(chrootdir):
            # give the child some time to fill its Redis keys
            sleeper = 1
            while True:
                if len(self.redis.keys()) > 0:
                    break
                sleeper += 1
                time.sleep(0.1)
>               assert sleeper < 10*15  # 15s
E               assert 150 < (10 * 15)
tests/test_modifyrepo.py:629: AssertionError
------------------------------ Captured log call -------------------------------
DEBUG    root:copr-repo:324 acquiring lock
DEBUG    filelock:_api.py:212 Attempting to acquire lock 72057592432461920 on /tmp/copr-test-copr-repok3h7000w/_@_tmp_@_copr-backend-test-710e3qgv_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:215 Lock 72057592432461920 acquired on /tmp/copr-test-copr-repok3h7000w/_@_tmp_@_copr-backend-test-710e3qgv_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    root:copr-repo:327 acquired lock
DEBUG    filelock:_api.py:244 Attempting to release lock 72057592432461920 on /tmp/copr-test-copr-repok3h7000w/_@_tmp_@_copr-backend-test-710e3qgv_@_john_@_empty_@_fedora-rawhide-x86_64.lock
DEBUG    filelock:_api.py:247 Lock 72057592432461920 released on /tmp/copr-test-copr-repok3h7000w/_@_tmp_@_copr-backend-test-710e3qgv_@_john_@_empty_@_fedora-rawhide-x86_64.lock
___________ TestLimitedWorkerManager.test_that_limits_are_respected ____________
self = <tests.test_worker_manager.TestLimitedWorkerManager object at 0xffffffa11e1bb0>
mc_time = <MagicMock name='time' id='72057592442851680'>
_mc_sleep = <MagicMock name='sleep' id='72057592442858688'>
caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffa0ed3ad0>
    @patch('copr_common.worker_manager.time.sleep')
    @patch('copr_common.worker_manager.time.time')
    def test_that_limits_are_respected(self, mc_time, _mc_sleep, caplog):
        # each time.time() call incremented by 1
        self.worker_manager.task_sleep = 5
        self.worker_manager.worker_timeout_start = 1000
        mc_time.side_effect = range(1000)
>       self.worker_manager.run(timeout=150)
tests/test_worker_manager.py:211: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/test_worker_manager.py:83: in run
    return super().run(*args, **kwargs)
/usr/lib/python3.12/site-packages/copr_common/worker_manager.py:475: in run
    self.log.debug("Reaped %s processes", self._clean_daemon_processes())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <tests.test_worker_manager.ToyWorkerManager object at 0xffffffa0ed3a70>
    def _clean_daemon_processes(self):
        """
        Check that we are not leaving any zombies behind us
        """
        waited = super()._clean_daemon_processes()
        self.log.debug("cleaned up %s, started %s", waited, self.started_in_cycle)
        if waited != self.started_in_cycle:
            if self.expected_terminations_in_cycle is not None:
                assert self.expected_terminations_in_cycle == waited
                return waited
>           assert False
E           assert False
tests/test_worker_manager.py:78: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
------------------------------ Captured log call -------------------------------
DEBUG    root:worker_manager.py:425 Worker.run() start at time 0
INFO     root:worker_manager.py:482 Starting worker worker:0, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113714 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:1, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113716 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:2, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113718 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:3, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113721 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:482 Starting worker worker:5, task.priority=0
DEBUG    root:worker_manager.py:575 background pid=1113723 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5'])
DEBUG    root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5'])
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2
DEBUG    root:worker_manager.py:513 Trying to clean old workers
INFO     root:worker_manager.py:525 Missing 'allocated' flag for worker worker:2
DEBUG    root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:513 Trying to clean old workers
DEBUG    root:worker_manager.py:454 No more tasks, waiting for workers
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113714
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113718
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113716
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113723
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113721
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113621
DEBUG    root:worker_manager.py:590 Worker Manager waited for pid=1113247
DEBUG    root:test_worker_manager.py:73 cleaned up 7, started 5
______________ TestWorkerManager.test_slow_priority_queue_filling ______________
self = <tests.test_worker_manager.TestWorkerManager object at 0xffffffa11e33b0>
    def test_slow_priority_queue_filling(self):
        """
        We discovered that adding tasks to a priority queue was a bottleneck
        when having a large (70k+ builds) queue, see #2095. Make sure this never
        happen again.
        """
        tasks = [ToyQueueTask(i) for i in range(100000)]
    
        # We need to run this test with logging only INFO, otherwise we waste
        # around 5 seconds just on running self.log.debug because we need to
        # connect to redis for each call
        # The point of this test is to make sure that adding tasks to priority
        # queue is not a bottleneck on production, and we don't use DEBUG there
        # anyway.
        log.setLevel(logging.INFO)
    
        t1 = time.time()
        for task in tasks:
            self.worker_manager.add_task(task)
        t2 = time.time()
    
        # It should actually be faster than 1 second but I am adding one to
        # prevent false alarms in case somebody has a slow machine
>       assert t2 - t1 < 2
E       assert (1701743423.1552684 - 1701743404.132487) < 2
tests/test_worker_manager.py:314: AssertionError
------------------------------ Captured log setup ------------------------------
DEBUG    root:worker_manager.py:386 Adding task 0 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 1 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 2 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 3 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 4 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 5 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 6 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 7 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 8 to queue, priority 0
DEBUG    root:worker_manager.py:386 Adding task 9 to queue, priority 0
=============================== warnings summary ===============================
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API
    warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning)
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870
  /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`.
  Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
    declare_namespace(pkg)
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108
  /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    override_loader = get_loader(override_package_name)
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
../../../../usr/lib64/python3.12/pkgutil.py:291
  /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead
    return find_loader(fullname)
tests/test_frontend.py:14
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_frontend.py:29
  /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:18
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/daemons/test_log.py:24
  /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:21
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:26
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:31
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/run/test_copr_prune_results.py:36
  /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated.
  Use @pytest.fixture instead; they are the same.
    @pytest.yield_fixture
tests/test_action.py: 5 warnings
tests/test_modifyrepo.py: 16 warnings
tests/run/test_copr_prune_results.py: 3 warnings
  /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior.
    warnings.warn(
tests/test_action.py::TestAction::test_comps_create
  /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
    now = datetime.utcnow()
tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker
  /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14.
    rv = reductor(4)
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True]
FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False]
FAILED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected
FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling
===== 5 failed, 171 passed, 11 skipped, 47 warnings in 3334.67s (0:55:34) ======
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process shutting down
+ cleanup
+ redis-cli -p 7777 shutdown
+ wait
error: 
RPM build errors:
Bad exit status from /var/tmp/rpm-tmp.ZtsnT6 (%check)
    Bad exit status from /var/tmp/rpm-tmp.ZtsnT6 (%check)
Child return code was: 1
EXCEPTION: [Error('Command failed: \n # bash --login -c /usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec\n', 1)]
Traceback (most recent call last):
  File "/usr/lib/python3.11/site-packages/mockbuild/trace_decorator.py", line 93, in trace
    result = func(*args, **kw)
             ^^^^^^^^^^^^^^^^^
  File "/usr/lib/python3.11/site-packages/mockbuild/util.py", line 597, in do_with_status
    raise exception.Error("Command failed: \n # %s\n%s" % (command, output), child.returncode)
mockbuild.exception.Error: Command failed: 
 # bash --login -c /usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec