Mock Version: 5.0 Mock Version: 5.0 Mock Version: 5.0 ENTER ['do_with_status'](['bash', '--login', '-c', '/usr/bin/rpmbuild -bs --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'], chrootPath='/var/lib/mock/f39-build-2185078-51286/root'env={'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;\\007"', 'PS1': ' \\s-\\v\\$ ', 'LANG': 'C.UTF-8'}shell=Falselogger=timeout=864000uid=996gid=135user='mockbuild'nspawn_args=[]unshare_net=TrueprintOutput=False) Executing command: ['bash', '--login', '-c', '/usr/bin/rpmbuild -bs --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'] with env {'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;\\007"', 'PS1': ' \\s-\\v\\$ ', 'LANG': 'C.UTF-8'} and shell False Building target platforms: noarch Building for target noarch setting SOURCE_DATE_EPOCH=1700697600 Wrote: /builddir/build/SRPMS/copr-backend-1.173-1.fc39.src.rpm Child return code was: 0 ENTER ['do_with_status'](['bash', '--login', '-c', '/usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'], chrootPath='/var/lib/mock/f39-build-2185078-51286/root'env={'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;\\007"', 'PS1': ' \\s-\\v\\$ ', 'LANG': 'C.UTF-8'}shell=Falselogger=timeout=864000uid=996gid=135user='mockbuild'nspawn_args=[]unshare_net=TrueprintOutput=False) Executing command: ['bash', '--login', '-c', '/usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec'] with env {'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOME': '/builddir', 'HOSTNAME': 'mock', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', 'PROMPT_COMMAND': 'printf "\\033]0;\\007"', 'PS1': ' \\s-\\v\\$ ', 'LANG': 'C.UTF-8'} and shell False Building target platforms: noarch Building for target noarch setting SOURCE_DATE_EPOCH=1700697600 Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.ul8Ehg + umask 022 + cd /builddir/build/BUILD + cd /builddir/build/BUILD + rm -rf copr-backend-1.173 + /usr/lib/rpm/rpmuncompress -x /builddir/build/SOURCES/copr-backend-1.173.tar.gz + STATUS=0 + '[' 0 -ne 0 ']' + cd copr-backend-1.173 + rm -rf /builddir/build/BUILD/copr-backend-1.173-SPECPARTS + /usr/bin/mkdir -p /builddir/build/BUILD/copr-backend-1.173-SPECPARTS + /usr/lib/rpm/rpmuncompress -x /builddir/build/SOURCES/test-data-copr-backend-5.tar.gz + STATUS=0 + '[' 0 -ne 0 ']' + /usr/bin/chmod -Rf a+rX,u+w,g-w,o-w . + RPM_EC=0 ++ jobs -p + exit 0 Executing(%build): /bin/sh -e /var/tmp/rpm-tmp.Q7NkN6 + umask 022 + cd /builddir/build/BUILD + CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + export CFLAGS + CXXFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + export CXXFLAGS + FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules ' + export FFLAGS + FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules ' + export FCFLAGS + VALAFLAGS=-g + export VALAFLAGS + RUSTFLAGS='-Copt-level=3 -Cdebuginfo=2 -Ccodegen-units=1 -Cstrip=none -Cforce-frame-pointers=yes -Clink-arg=-Wl,-z,relro -Clink-arg=-Wl,-z,now --cap-lints=warn' + export RUSTFLAGS + LDFLAGS='-Wl,-z,relro -Wl,--as-needed -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -Wl,--build-id=sha1 ' + export LDFLAGS + LT_SYS_LIBRARY_PATH=/usr/lib: + export LT_SYS_LIBRARY_PATH + CC=gcc + export CC + CXX=g++ + export CXX + cd copr-backend-1.173 + make -C docs -j8 html make: Entering directory '/builddir/build/BUILD/copr-backend-1.173/docs' sphinx-build-3 -b html -d build/doctrees source build/html Running Sphinx v6.2.1 making output directory... done building [mo]: targets for 0 po files that are out of date writing output... building [html]: targets for 32 source files that are out of date updating environment: [new config] 32 added, 0 changed, 0 removed reading sources... [ 3%] Autodoc reading sources... [ 6%] Backend_design reading sources... [ 9%] Configuration reading sources... [ 12%] INSTALL reading sources... [ 15%] README reading sources... [ 18%] Statistics reading sources... [ 21%] VmManagement reading sources... [ 25%] index reading sources... [ 28%] package/actions reading sources... [ 31%] package/constants reading sources... [ 34%] package/createrepo reading sources... [ 37%] package/daemons/backend reading sources... [ 40%] package/daemons/dispatcher reading sources... [ 43%] package/daemons/job_grab reading sources... [ 46%] package/daemons/log reading sources... [ 50%] package/daemons/vm_master reading sources... [ 53%] package/exceptions reading sources... [ 56%] package/frontend reading sources... [ 59%] package/helpers reading sources... [ 62%] package/job reading sources... [ 65%] package/mockremote/__init__ reading sources... [ 68%] package/mockremote/builder reading sources... [ 71%] package/sign reading sources... [ 75%] package/vm_manage/__init__ reading sources... [ 78%] package/vm_manage/check reading sources... [ 81%] package/vm_manage/event_handle reading sources... [ 84%] package/vm_manage/executor reading sources... [ 87%] package/vm_manage/manager reading sources... [ 90%] package/vm_manage/models reading sources... [ 93%] package/vm_manage/spawn reading sources... [ 96%] package/vm_manage/terminate reading sources... [100%] run_scripts WARNING: autodoc: failed to import module 'actions' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'constants' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'createrepo' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'daemons.backend' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'daemons.dispatcher' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'daemons.job_grab' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'daemons.log' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'daemons.vm_master' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'exceptions' from module 'backend'; the following exception was raised: No module named 'backend' looking for now-outdated files... none found pickling environment... WARNING: autodoc: failed to import module 'frontend' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'helpers' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'job' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'mockremote.__init__' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'mockremote.builder' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'sign' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.__init__' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.check' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.event_handle' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.executor' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.manager' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.models' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.spawn' from module 'backend'; the following exception was raised: No module named 'backend' WARNING: autodoc: failed to import module 'vm_manage.terminate' from module 'backend'; the following exception was raised: No module named 'backend' done checking consistency... done preparing documents... done writing output... [ 3%] Autodoc writing output... [ 6%] Backend_design writing output... [ 9%] Configuration writing output... [ 12%] INSTALL writing output... [ 15%] README writing output... [ 18%] Statistics writing output... [ 21%] VmManagement writing output... [ 25%] index writing output... [ 28%] package/actions writing output... [ 31%] package/constants writing output... [ 34%] package/createrepo writing output... [ 37%] package/daemons/backend writing output... [ 40%] package/daemons/dispatcher writing output... [ 43%] package/daemons/job_grab writing output... [ 46%] package/daemons/log writing output... [ 50%] package/daemons/vm_master writing output... [ 53%] package/exceptions writing output... [ 56%] package/frontend writing output... [ 59%] package/helpers writing output... [ 62%] package/job writing output... [ 65%] package/mockremote/__init__ writing output... [ 68%] package/mockremote/builder writing output... [ 71%] package/sign writing output... [ 75%] package/vm_manage/__init__ writing output... [ 78%] package/vm_manage/check writing output... [ 81%] package/vm_manage/event_handle writing output... [ 84%] package/vm_manage/executor writing output... [ 87%] package/vm_manage/manager writing output... [ 90%] package/vm_manage/models writing output... [ 93%] package/vm_manage/spawn writing output... [ 96%] package/vm_manage/terminate writing output... [100%] run_scripts generating indices... genindex done highlighting module code... writing additional pages... search done copying static files... done copying extra files... done dumping search index in English (code: en)... done dumping object inventory... done build succeeded, 23 warnings. The HTML pages are in build/html. Build finished. The HTML pages are in build/html. make: Leaving directory '/builddir/build/BUILD/copr-backend-1.173/docs' + CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + LDFLAGS='-Wl,-z,relro -Wl,--as-needed -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -Wl,--build-id=sha1 ' + /usr/bin/python3 setup.py build '--executable=/usr/bin/python3 -sP' running build running build_py creating build creating build/lib creating build/lib/copr_backend copying copr_backend/setup.py -> build/lib/copr_backend copying copr_backend/hitcounter.py -> build/lib/copr_backend copying copr_backend/dispatcher.py -> build/lib/copr_backend copying copr_backend/background_worker_build.py -> build/lib/copr_backend copying copr_backend/app.py -> build/lib/copr_backend copying copr_backend/sshcmd.py -> build/lib/copr_backend copying copr_backend/cancellable_thread.py -> build/lib/copr_backend copying copr_backend/createrepo.py -> build/lib/copr_backend copying copr_backend/exceptions.py -> build/lib/copr_backend copying copr_backend/frontend.py -> build/lib/copr_backend copying copr_backend/constants.py -> build/lib/copr_backend copying copr_backend/vm_alloc.py -> build/lib/copr_backend copying copr_backend/worker_manager.py -> build/lib/copr_backend copying copr_backend/rpm_builds.py -> build/lib/copr_backend copying copr_backend/__init__.py -> build/lib/copr_backend copying copr_backend/helpers.py -> build/lib/copr_backend copying copr_backend/ans_utils.py -> build/lib/copr_backend copying copr_backend/background_worker.py -> build/lib/copr_backend copying copr_backend/msgbus.py -> build/lib/copr_backend copying copr_backend/job.py -> build/lib/copr_backend copying copr_backend/sign.py -> build/lib/copr_backend copying copr_backend/actions.py -> build/lib/copr_backend creating build/lib/copr_backend/stats_templates copying copr_backend/stats_templates/__init__.py -> build/lib/copr_backend/stats_templates creating build/lib/copr_backend/daemons copying copr_backend/daemons/action_dispatcher.py -> build/lib/copr_backend/daemons copying copr_backend/daemons/build_dispatcher.py -> build/lib/copr_backend/daemons copying copr_backend/daemons/__init__.py -> build/lib/copr_backend/daemons copying copr_backend/daemons/log.py -> build/lib/copr_backend/daemons running egg_info creating copr_backend.egg-info writing copr_backend.egg-info/PKG-INFO writing dependency_links to copr_backend.egg-info/dependency_links.txt writing top-level names to copr_backend.egg-info/top_level.txt writing manifest file 'copr_backend.egg-info/SOURCES.txt' reading manifest file 'copr_backend.egg-info/SOURCES.txt' adding license file 'LICENSE' writing manifest file 'copr_backend.egg-info/SOURCES.txt' copying copr_backend/stats_templates/arches.html.j2 -> build/lib/copr_backend/stats_templates copying copr_backend/stats_templates/chroots.html.j2 -> build/lib/copr_backend/stats_templates copying copr_backend/stats_templates/distro.html.j2 -> build/lib/copr_backend/stats_templates copying copr_backend/stats_templates/index.html.j2 -> build/lib/copr_backend/stats_templates copying copr_backend/stats_templates/topmost.html.j2 -> build/lib/copr_backend/stats_templates + RPM_EC=0 ++ jobs -p + exit 0 Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.gAcBDd + umask 022 + cd /builddir/build/BUILD + '[' /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch '!=' / ']' + rm -rf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch ++ dirname /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch + mkdir -p /builddir/build/BUILDROOT + mkdir /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch + CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + export CFLAGS + CXXFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + export CXXFLAGS + FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules ' + export FFLAGS + FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules ' + export FCFLAGS + VALAFLAGS=-g + export VALAFLAGS + RUSTFLAGS='-Copt-level=3 -Cdebuginfo=2 -Ccodegen-units=1 -Cstrip=none -Cforce-frame-pointers=yes -Clink-arg=-Wl,-z,relro -Clink-arg=-Wl,-z,now --cap-lints=warn' + export RUSTFLAGS + LDFLAGS='-Wl,-z,relro -Wl,--as-needed -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -Wl,--build-id=sha1 ' + export LDFLAGS + LT_SYS_LIBRARY_PATH=/usr/lib: + export LT_SYS_LIBRARY_PATH + CC=gcc + export CC + CXX=g++ + export CXX + cd copr-backend-1.173 + CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + LDFLAGS='-Wl,-z,relro -Wl,--as-needed -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -Wl,--build-id=sha1 ' + /usr/bin/python3 setup.py install -O1 --skip-build --root /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch --prefix /usr running install /usr/lib/python3.12/site-packages/setuptools/_distutils/cmd.py:66: SetuptoolsDeprecationWarning: setup.py install is deprecated. !! ******************************************************************************** Please avoid running ``setup.py`` directly. Instead, use pypa/build, pypa/installer, pypa/build or other standards-based tools. Follow the current Python packaging guidelines when building Python RPM packages. See https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html and https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/ for details. ******************************************************************************** !! self.initialize_options() running install_lib creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12 creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/setup.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates copying build/lib/copr_backend/stats_templates/index.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates copying build/lib/copr_backend/stats_templates/chroots.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates copying build/lib/copr_backend/stats_templates/topmost.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates copying build/lib/copr_backend/stats_templates/distro.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates copying build/lib/copr_backend/stats_templates/arches.html.j2 -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates copying build/lib/copr_backend/stats_templates/__init__.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates creating /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons copying build/lib/copr_backend/daemons/action_dispatcher.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons copying build/lib/copr_backend/daemons/build_dispatcher.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons copying build/lib/copr_backend/daemons/__init__.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons copying build/lib/copr_backend/daemons/log.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons copying build/lib/copr_backend/hitcounter.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/dispatcher.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/background_worker_build.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/app.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/sshcmd.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/cancellable_thread.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/createrepo.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/exceptions.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/frontend.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/constants.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/vm_alloc.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/worker_manager.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/rpm_builds.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/__init__.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/helpers.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/ans_utils.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/background_worker.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/msgbus.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/job.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/sign.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend copying build/lib/copr_backend/actions.py -> /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/setup.py to setup.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/stats_templates/__init__.py to __init__.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/action_dispatcher.py to action_dispatcher.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/build_dispatcher.py to build_dispatcher.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/__init__.py to __init__.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/daemons/log.py to log.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/hitcounter.py to hitcounter.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/dispatcher.py to dispatcher.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/background_worker_build.py to background_worker_build.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/app.py to app.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/sshcmd.py to sshcmd.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/cancellable_thread.py to cancellable_thread.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/createrepo.py to createrepo.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/exceptions.py to exceptions.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/frontend.py to frontend.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/constants.py to constants.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/vm_alloc.py to vm_alloc.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/worker_manager.py to worker_manager.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/rpm_builds.py to rpm_builds.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/__init__.py to __init__.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/helpers.py to helpers.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/ans_utils.py to ans_utils.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/background_worker.py to background_worker.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/msgbus.py to msgbus.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/job.py to job.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/sign.py to sign.cpython-312.pyc byte-compiling /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend/actions.py to actions.cpython-312.pyc writing byte-compilation script '/tmp/tmph7xxay2h.py' /usr/bin/python3 /tmp/tmph7xxay2h.py removing /tmp/tmph7xxay2h.py running install_egg_info running egg_info writing copr_backend.egg-info/PKG-INFO writing dependency_links to copr_backend.egg-info/dependency_links.txt writing top-level names to copr_backend.egg-info/top_level.txt reading manifest file 'copr_backend.egg-info/SOURCES.txt' adding license file 'LICENSE' writing manifest file 'copr_backend.egg-info/SOURCES.txt' Copying copr_backend.egg-info to /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12/site-packages/copr_backend-1.173-py3.12.egg-info running install_scripts + rm -rfv /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/bin/__pycache__ + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/var/lib/copr/public_html/results + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/lighttpd/ + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/copr + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/logrotate.d/ + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/systemd/system + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//var/log/copr-backend + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//var/run/copr-backend/ + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/lib/tmpfiles.d + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/sbin + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.daily + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.weekly + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/sudoers.d + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/bin/ + cp -a copr-backend-service /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/sbin/ + cp -a run/copr-assure-permissions run/copr-aws-s3-hitcounter run/copr-backend-analyze-results run/copr-backend-generate-graphs run/copr-backend-process-action run/copr-backend-process-build run/copr-backend-resultdir-cleaner run/copr-backend-unknown-resalloc-tickets.py run/copr-compress-live-logs run/copr-rename-chroot run/copr-repo run/copr-run-dispatcher run/copr_find_obsolete_builds.sh run/copr_find_wrong_chroot_artifacts.py run/copr_fix_gpg.py run/copr_log_hitcounter.py run/copr_print_results_to_delete.py run/copr_prune_results.py run/copr_prune_srpms.py run/copr_run_logger.py run/copr_sign_unsigned.py run/print_queues.py /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/bin/ + cp -a conf/copr-be.conf.example /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/copr/copr-be.conf + install -p -m 755 conf/crontab/daily /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.daily/copr-backend + install -p -m 755 conf/crontab/weekly /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/cron.weekly/copr-backend + cp -a conf/lighttpd/dirlisting.conf conf/lighttpd/mime.conf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/lighttpd/ + cp -a conf/logrotate/copr-backend /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/logrotate.d/ + cp -a conf/tmpfiles.d/copr-backend.conf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/lib/tmpfiles.d + touch /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/var/log/copr-backend/copr.log + touch /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/var/log/copr-backend/prune_old.log + cp -a units/copr-backend.target units/copr-backend-action.service units/copr-backend-build.service units/copr-backend-log.service /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch//usr/lib/systemd/system/ + install -m 0644 conf/copr.sudoers.d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/sudoers.d/copr + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/etc/logstash.d + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/logstash/patterns/ + cp -a conf/logstash/lighttpd.pattern /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/logstash/patterns/lighttpd.pattern + install -d /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/examples//etc/logstash.d + cp -a conf/logstash/copr_backend.conf /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/examples//etc/logstash.d/copr_backend.conf + cp -a docs/build/html /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/share/doc/copr-backend/ + /usr/bin/find-debuginfo -j8 --strict-build-id -m -i --build-id-seed 1.173-1.fc39 --unique-debug-suffix -1.173-1.fc39.noarch --unique-debug-src-base copr-backend-1.173-1.fc39.noarch --run-dwz --dwz-low-mem-die-limit 10000000 --dwz-max-die-limit 50000000 -S debugsourcefiles.list /builddir/build/BUILD/copr-backend-1.173 find-debuginfo: starting Extracting debug info from 0 files Creating .debug symlinks for symlinks to ELF files find: ‘debug’: No such file or directory find-debuginfo: done + /usr/lib/rpm/check-buildroot + /usr/lib/rpm/redhat/brp-ldconfig + /usr/lib/rpm/brp-compress + /usr/lib/rpm/redhat/brp-strip-lto /usr/bin/strip + /usr/lib/rpm/brp-strip-static-archive /usr/bin/strip + /usr/lib/rpm/check-rpaths + /usr/lib/rpm/redhat/brp-mangle-shebangs mangling shebang in /usr/bin/copr_find_obsolete_builds.sh from /bin/bash to #!/usr/bin/bash mangling shebang in /usr/bin/copr-compress-live-logs from /bin/sh to #!/usr/bin/sh mangling shebang in /usr/sbin/copr-backend-service from /bin/sh to #!/usr/bin/sh mangling shebang in /etc/cron.weekly/copr-backend from /bin/sh to #!/usr/bin/sh + /usr/lib/rpm/brp-remove-la-files + env /usr/lib/rpm/redhat/brp-python-bytecompile '' 1 0 -j8 Bytecompiling .py files below /builddir/build/BUILDROOT/copr-backend-1.173-1.fc39.noarch/usr/lib/python3.12 using python3.12 + /usr/lib/rpm/redhat/brp-python-hardlink Executing(%check): /bin/sh -e /var/tmp/rpm-tmp.wYKJ8U + umask 022 + cd /builddir/build/BUILD + CFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + export CFLAGS + CXXFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer ' + export CXXFLAGS + FFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules ' + export FFLAGS + FCFLAGS='-O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Wp,-U_FORTIFY_SOURCE,-D_FORTIFY_SOURCE=3 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fno-omit-frame-pointer -I/usr/lib/gfortran/modules ' + export FCFLAGS + VALAFLAGS=-g + export VALAFLAGS + RUSTFLAGS='-Copt-level=3 -Cdebuginfo=2 -Ccodegen-units=1 -Cstrip=none -Cforce-frame-pointers=yes -Clink-arg=-Wl,-z,relro -Clink-arg=-Wl,-z,now --cap-lints=warn' + export RUSTFLAGS + LDFLAGS='-Wl,-z,relro -Wl,--as-needed -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -Wl,--build-id=sha1 ' + export LDFLAGS + LT_SYS_LIBRARY_PATH=/usr/lib: + export LT_SYS_LIBRARY_PATH + CC=gcc + export CC + CXX=g++ + export CXX + cd copr-backend-1.173 + ./run_tests.sh -vv --no-cov + set -e ++ dirname ./run_tests.sh + srcdir=. ++ grep -E '%global[[:space:]]*tests_version' ++ awk '{ print $3 }' + test_tarball_version=5 ++ grep -E '%global[[:space:]]*tests_tar' ++ awk '{ print $3 }' + test_tarball_name=test-data-copr-backend + test_tarball_extracted=test-data-copr-backend-5 + test_tarball=test-data-copr-backend-5.tar.gz + test -d test-data-copr-backend-5 + export TEST_DATA_DIRECTORY ++ readlink -f test-data-copr-backend-5 + TEST_DATA_DIRECTORY=/builddir/build/BUILD/copr-backend-1.173/test-data-copr-backend-5 + REDIS_PORT=7777 + trap cleanup EXIT + redis-server --port 7777 ++ readlink -f ../common + common_path=/builddir/build/BUILD/common ++ readlink -f ../messaging + messaging_path=/builddir/build/BUILD/messaging + export PYTHONPATH=/builddir/build/BUILD/common:/builddir/build/BUILD/messaging:/builddir/build/BUILD/copr-backend-1.173:/builddir/build/BUILD/copr-backend-1.173/tests:/builddir/build/BUILD/copr-backend-1.173/run + PYTHONPATH=/builddir/build/BUILD/common:/builddir/build/BUILD/messaging:/builddir/build/BUILD/copr-backend-1.173:/builddir/build/BUILD/copr-backend-1.173/tests:/builddir/build/BUILD/copr-backend-1.173/run + export PATH=/builddir/build/BUILD/copr-backend-1.173/run:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/sbin + PATH=/builddir/build/BUILD/copr-backend-1.173/run:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/sbin + COVPARAMS=(--cov-report term-missing --cov ./copr_backend --cov ./run) + KEEP_ARGS=() + for arg in "$@" + case $arg in + KEEP_ARGS+=("$arg") + for arg in "$@" + case $arg in + COVPARAMS=() + python3 -m pytest -s tests -vv ============================= test session starts ============================== platform linux -- Python 3.12.0, pytest-7.3.2, pluggy-1.2.0 -- /usr/bin/python3 cachedir: .pytest_cache rootdir: /builddir/build/BUILD/copr-backend-1.173 collecting ... collected 187 items tests/test_action.py::TestAction::test_action_run_legal_flag PASSED tests/test_action.py::TestAction::test_action_handle_forks PASSED tests/test_action.py::TestAction::test_action_run_rename SKIPPED (Fixme, test doesn't work.) tests/test_action.py::TestAction::test_action_run_rename_success_on_empty_src SKIPPED (Fixme, test doesn't work.) tests/test_action.py::TestAction::test_action_run_rename_failure_on_destination_exists SKIPPED (Fixme, test doesn't work.) tests/test_action.py::TestAction::test_action_run_delete_copr PASSED tests/test_action.py::TestAction::test_action_run_delete_copr_remove_folders SKIPPED (Fixme, test doesn't work.) tests/test_action.py::TestAction::test_delete_no_chroot_dirs PASSED tests/test_action.py::TestAction::test_delete_build_succeeded PASSED tests/test_action.py::TestAction::test_delete_build_acr_reflected[False] PASSED tests/test_action.py::TestAction::test_delete_build_acr_reflected[True] PASSED tests/test_action.py::TestAction::test_delete_build_succeeded_createrepo_error PASSED tests/test_action.py::TestAction::test_delete_two_chroots PASSED tests/test_action.py::TestAction::test_delete_two_chroots_two_remain PASSED tests/test_action.py::TestAction::test_delete_build_with_bad_pkg_name PASSED tests/test_action.py::TestAction::test_delete_multiple_builds_succeeded PASSED tests/test_action.py::TestAction::test_handle_createrepo_ok[False] PASSED tests/test_action.py::TestAction::test_handle_createrepo_ok[True] PASSED tests/test_action.py::TestAction::test_handle_createrepo_failure_1 PASSED tests/test_action.py::TestAction::test_handle_generate_gpg_key SKIPPED (Fixme, test doesn't work.) tests/test_action.py::TestAction::test_request_exception_is_taken_care_of_when_posting_to_frontend SKIPPED (Fixme, test doesn't work.) tests/test_action.py::TestAction::test_delete_chroot PASSED tests/test_action.py::TestAction::test_comps_create PASSED tests/test_action.py::TestAction::test_remove_dirs PASSED tests/test_background_worker_build.py::test_waiting_for_repo_fail PASSED tests/test_background_worker_build.py::test_waiting_for_repo_success PASSED tests/test_background_worker_build.py::test_full_rpm_build_no_sign PASSED tests/test_background_worker_build.py::test_prev_build_backup PASSED tests/test_background_worker_build.py::test_full_srpm_build PASSED tests/test_background_worker_build.py::test_build_and_sign PASSED tests/test_background_worker_build.py::test_sign_built_packages_exception PASSED tests/test_background_worker_build.py::test_unexpected_exception PASSED tests/test_background_worker_build.py::test_build_info_file_failure PASSED tests/test_background_worker_build.py::test_invalid_job_info PASSED tests/test_background_worker_build.py::test_cancel_build_on_vm_allocation PASSED tests/test_background_worker_build.py::test_cancel_build_on_tail_log_no_ssh PASSED tests/test_background_worker_build.py::test_cancel_before_vm PASSED tests/test_background_worker_build.py::test_cancel_before_start PASSED tests/test_background_worker_build.py::test_build_retry PASSED tests/test_background_worker_build.py::test_fe_disallowed_start PASSED tests/test_background_worker_build.py::test_fe_failed_start PASSED tests/test_background_worker_build.py::test_cancel_script_failure PASSED tests/test_background_worker_build.py::test_cancel_build_during_log_download PASSED tests/test_background_worker_build.py::test_ssh_connection_error PASSED tests/test_background_worker_build.py::test_average_step PASSED tests/test_background_worker_build.py::test_retry_for_ssh_tail_failure PASSED tests/test_background_worker_build.py::test_build_failure PASSED tests/test_background_worker_build.py::test_createrepo_failure PASSED tests/test_background_worker_build.py::test_existing_compressed_file PASSED tests/test_background_worker_build.py::test_tail_f_nonzero_exit PASSED tests/test_background_worker_build.py::test_wrong_copr_rpmbuild_daemon_output PASSED tests/test_background_worker_build.py::test_unable_to_start_builder PASSED tests/test_background_worker_build.py::test_retry_vm_factory_take PASSED tests/test_background_worker_build.py::test_failed_build_retry PASSED tests/test_background_worker_build.py::test_buildjob_tags PASSED tests/test_background_worker_build.py::test_task_url PASSED tests/test_build_dispatcher.py::test_priority_numbers PASSED tests/test_build_dispatcher.py::test_frontend_priority[True-2000000] PASSED tests/test_build_dispatcher.py::test_frontend_priority[False-0] PASSED tests/test_build_dispatcher.py::test_sandbox_priority PASSED tests/test_config_reader.py::TestBackendConfigReader::test_minimal_file_and_defaults PASSED tests/test_config_reader.py::TestBackendConfigReader::test_correct_build_limits PASSED tests/test_config_reader.py::TestBackendConfigReader::test_invalid_limits[builds_max_workers=asdfa\n] PASSED tests/test_config_reader.py::TestBackendConfigReader::test_invalid_limits[actions_max_workers=asdfa\n] PASSED tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc\n] PASSED tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc=asdf\n] PASSED tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc=10=\n] PASSED tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_arch=abc=1,\n] PASSED tests/test_config_reader.py::TestBackendConfigReader::test_invalid_build_limits[builds_max_workers_tag=abc=1,\n] PASSED tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_normal PASSED tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_already_done PASSED tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_other_already_done PASSED tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_devel_mismatch PASSED tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_full_we_take_others PASSED tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_full_others_take_us PASSED tests/test_createrepo.py::TestBatchedCreaterepo::test_batched_createrepo_task_limit PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend[get] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend[post] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend[put] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_wrappers[get] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_wrappers[post] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_wrappers[put] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_first_try_ok PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_second_try_ok[get] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_second_try_ok[post] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_second_try_ok[put] PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_err_400 PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_all_attempts_failed PASSED tests/test_frontend.py::TestFrontendClient::test_post_to_frontend_repeated_indefinitely PASSED tests/test_frontend.py::TestFrontendClient::test_retries_on_outdated_frontend PASSED tests/test_frontend.py::TestFrontendClient::test_update PASSED tests/test_frontend.py::TestFrontendClient::test_starting_build PASSED tests/test_frontend.py::TestFrontendClient::test_starting_build_err PASSED tests/test_frontend.py::TestFrontendClient::test_starting_build_err_2 PASSED tests/test_frontend.py::TestFrontendClient::test_reschedule_build PASSED tests/test_helpers.py::TestHelpers::test_redis_logger_exception PASSED tests/test_helpers.py::TestHelpers::test_get_chroot_arch PASSED tests/test_helpers.py::TestHelpers::test_format_filename PASSED tests/test_helpers.py::TestHelpers::test_walk_limited PASSED tests/test_helpers.py::TestHelpers::test_recursive_copy_and_link_rpms PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_add_subdir PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_createrepo PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_already_processed PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_two_builds PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_full PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_batched_others_full PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_add_del_mixup PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_add_subdir_devel PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add0] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add1] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add2] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_validator[add3] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_subdir_none_doesnt_raise PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_do_stat PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[True-fedora-37---no-database] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[False-centos-stream-8---no-database] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[True-rhel-8---no-database] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_run_createrepo[False-epel-8---no-database] PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_el5 SKIPPED (createrepo_c dropped md5 checksum support) tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_noop PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_in_call PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_passes PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_passes_2 PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_rpms_to_remove_passes_3 PASSED tests/test_modifyrepo.py::TestModifyRepo::test_comps_present PASSED tests/test_modifyrepo.py::TestModifyRepo::test_run_prunerepo SKIPPED (Works locally and in Copr but fails in Koji) tests/test_modifyrepo.py::TestModifyRepo::test_pruner_arg_max PASSED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED tests/test_modifyrepo.py::test_aws_cdn_refresh FAILEDPASSED tests/test_sign.py::TestSign::test_get_pubkey PASSED tests/test_sign.py::TestSign::test_get_pubkey_error PASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_key PASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_error PASSED tests/test_sign.py::TestSign::test_get_pubkey_outfile tests/test_modifyrepo.py::test_aws_cdn_refresh PASSED tests/test_sign.py::TestSign::test_sign_one PASSED tests/test_sign.py::TestSign::test_sign_one_popen_error PASSEDPASSED tests/test_sign.py::TestSign::test_sign_one_cmd_erro tests/test_sign.py::TestSign::test_get_pubkey PASSED tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly PASSED tests/test_sign.py::TestSign::test_get_pubkey_error PASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_key PASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_error PASSED tests/test_sign.py::TestSign::test_get_pubkey_outfile PASSED tests/test_sign.py::TestSign::test_sign_one PASSED tests/test_sign.py::TestSign::test_sign_one_popen_error PASSED tests/test_sign.py::TestSign::test_sign_one_cmd_erro PASSED tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly PASSED tests/test_sign.py::TestSign::test_create_user_keys PASSED tests/test_sign.py::TestSign::test_create_user_keys_error_1 PASSED tests/test_sign.py::TestSign::test_create_user_keys_err PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one PASSEDPASSED tests/test_sign.py::TestSign::test_create_user_keys tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all PASSED tests/test_sign.py::TestSign::test_create_user_keys_error_1 PASSED tests/test_sign.py::TestSign::test_create_user_keys_err PASSED tests/test_sign.py::test_chroot_gpg_hashes PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSED tests/test_sshcmd.py::test_ipv4_ipv6_rsync PASSEDPASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok tests/test_vm_alloc.py::test_ticket PASSEDPASSED tests/test_vm_alloc.py::test_ticket_with_args tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_raises PASSEDPASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSEDPASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all tests/test_worker_limits.py::test_predicate_worker_limit PASSED tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSEDPASSED tests/test_worker_limits.py::test_group_worker_limit PASSED tests/test_sign.py::test_chroot_gpg_hashes tests/test_worker_limits.py::test_worker_limit_info PASSEDPASSED tests/test_worker_limits.py::test_string_counter tests/test_sshcmd.py::test_ipv4_ipv6_rsync PASSEDPASSED tests/test_worker_manager.py::TestPrioQueue::test_queue_order tests/test_vm_alloc.py::test_ticket PASSED tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSEDFAILED tests/test_worker_manager.py::TestPrioQueue::test_push_back PASSEDPASSED tests/test_vm_alloc.py::test_ticket_with_args tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_raises tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] ERRORPASSED tests/test_modifyrepo.py::test_aws_cdn_refresh tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSED tests/test_worker_limits.py::test_predicate_worker_limit PASSED tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSED tests/test_worker_limits.py::test_group_worker_limit PASSEDPASSED tests/test_worker_limits.py::test_worker_limit_info tests/test_sign.py::TestSign::test_get_pubkey PASSED tests/test_worker_limits.py::test_string_counter PASSEDPASSED tests/test_worker_manager.py::TestPrioQueue::test_queue_order tests/test_sign.py::TestSign::test_get_pubkey_error PASSED tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSEDPASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_key tests/test_worker_manager.py::TestPrioQueue::test_push_back PASSED tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected PASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_error PASSED tests/test_sign.py::TestSign::test_get_pubkey_outfile FAILEDPASSED tests/test_sign.py::TestSign::test_sign_one PASSED tests/test_sign.py::TestSign::test_sign_one_popen_error PASSED tests/test_sign.py::TestSign::test_sign_one_cmd_erro tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] ERROR tests/test_modifyrepo.py::test_aws_cdn_refresh PASSED tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly PASSED tests/test_sign.py::TestSign::test_get_pubkey PASSED tests/test_sign.py::TestSign::test_get_pubkey_error PASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_key PASSED tests/test_sign.py::TestSign::test_get_pubkey_unknown_error PASSED tests/test_sign.py::TestSign::test_get_pubkey_outfile PASSED tests/test_sign.py::TestSign::test_sign_one PASSED tests/test_sign.py::TestSign::test_sign_one_popen_error PASSED tests/test_sign.py::TestSign::test_sign_one_cmd_erro PASSED tests/test_sign.py::TestSign::test_call_sign_bin_repeatedly PASSEDPASSED tests/test_sign.py::TestSign::test_create_user_keys tests/test_sign.py::TestSign::test_create_user_keys PASSEDPASSED tests/test_sign.py::TestSign::test_create_user_keys_error_1 PASSED tests/test_sign.py::TestSign::test_create_user_keys_error_1 tests/test_sign.py::TestSign::test_create_user_keys_err PASSED tests/test_sign.py::TestSign::test_create_user_keys_err PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_nothing PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_ok PASSEDPASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey tests/test_sign.py::TestSign::test_sign_rpms_id_dir_error_on_pubkey PASSEDPASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key tests/test_sign.py::TestSign::test_sign_rpms_id_dir_no_pub_key PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one PASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all PASSED tests/test_sign.py::test_chroot_gpg_hashes PASSEDPASSED tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_one tests/test_sshcmd.py::test_ipv4_ipv6_rsync PASSED tests/test_vm_alloc.py::test_ticket PASSED tests/test_vm_alloc.py::test_ticket_with_args PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSEDPASSED tests/test_vm_alloc.py::test_ticket_wait_ready_raises tests/test_sign.py::TestSign::test_sign_rpms_id_dir_sign_error_all PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSED tests/test_worker_limits.py::test_predicate_worker_limit PASSED tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSED tests/test_worker_limits.py::test_group_worker_limit PASSEDPASSED tests/test_worker_limits.py::test_worker_limit_info PASSED tests/test_sign.py::test_chroot_gpg_hashes PASSED tests/test_worker_limits.py::test_string_counter tests/test_sshcmd.py::test_ipv4_ipv6_rsync PASSED tests/test_worker_manager.py::TestPrioQueue::test_queue_order PASSED tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSEDPASSED tests/test_worker_manager.py::TestPrioQueue::test_push_back tests/test_vm_alloc.py::test_ticket PASSED tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected PASSED tests/test_vm_alloc.py::test_ticket_with_args PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_normal PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_raises PASSED tests/test_vm_alloc.py::test_ticket_wait_ready_fallback PASSED tests/test_worker_limits.py::test_predicate_worker_limit PASSED tests/test_worker_limits.py::test_predicate_worker_limit_sometimes PASSED tests/test_worker_limits.py::test_group_worker_limit PASSED tests/test_worker_limits.py::test_worker_limit_info PASSED tests/test_worker_limits.py::test_string_counter PASSED tests/test_worker_manager.py::TestPrioQueue::test_queue_order PASSED tests/test_worker_manager.py::TestPrioQueue::test_pop_push PASSED tests/test_worker_manager.py::TestPrioQueue::test_push_back PASSED tests/test_worker_manager.py::TestPrioQueue::test_shift_prio PASSED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected FAILED tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling PASSED tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect FAILED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities PASSED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED tests/run/test_copr_prune_results.py::TestPruneResults::test_run FAILED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities PASSED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED tests/run/test_copr_prune_results.py::TestPruneResults::test_run SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_main SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_main SKIPPED (Fixme or remove, test doesn't work.) ==================================== ERRORS ==================================== __ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[False] ___ + Exception Group Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call | result: Optional[TResult] = func() | ^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in | lambda: ihook(item=item, **kwds), when=when, reraise=reraise | ^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__ | return self._hookexec(self.name, self._hookimpls, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec | return self._inner_hookexec(hook_name, methods, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall | return outcome.get_result() | ^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result | raise exc.with_traceback(exc.__traceback__) | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall | res = hook_impl.function(*args) | ^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown | item.session._setupstate.teardown_exact(nextitem) | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact | raise exceptions[0] | ExceptionGroup: errors while tearing down (2 sub-exceptions) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture | _call_with_optional_argument(func, method) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument | func(arg) | File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method | shutil.rmtree(self.workdir) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repomxlyevwj' +---------------- 2 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory | shutil.rmtree(directory) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-0jbratni' +------------------------------------ =================================== FAILURES =================================== __________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________ self = def test_copr_modifyrepo_locks(self): with _lock(self.workdir) as opts: cmd = [modifyrepo, opts.directory, '--log-to-stdout'] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: proc.communicate(timeout=2) assert 0 # this shouldn't happen except subprocess.TimeoutExpired: proc.kill() out, err = proc.communicate() assert b"acquired lock" not in out assert b"acquired lock" not in err # with released lock proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) > out, err = proc.communicate(timeout=5) tests/test_modifyrepo.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib64/python3.12/subprocess.py:1209: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.12/subprocess.py:2109: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = endtime = 2353053.97806015, orig_timeout = 5, stdout_seq = [], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-repo4lizes5f', '--log-to-stdout']' timed out after 5 seconds /usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:215 Lock 72057592818997616 acquired on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:247 Lock 72057592818997616 released on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock _____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________ self = f_second_build = run_bg = True @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) # sys.exit() can not be used in testsuite os._exit(0) # pylint: disable=protected-access with _lock(chrootdir): # give the child some time to fill its Redis keys sleeper = 1 while True: if len(self.redis.keys()) > 0: break sleeper += 1 time.sleep(0.1) > assert sleeper < 10*15 # 15s E assert 150 < (10 * 15) tests/test_modifyrepo.py:629: AssertionError ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592816893104 on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:215 Lock 72057592816893104 acquired on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592816893104 on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:247 Lock 72057592816893104 released on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock _____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________ self = f_second_build = run_bg = False @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. > assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) E AssertionError: assert False E + where False = call_copr_repo('/tmp/copr-backend-test-0jbratni/john/empty/fedora-rawhide-x86_64', add=['00000002-example']) tests/test_modifyrepo.py:617: AssertionError ______________ TestWorkerManager.test_slow_priority_queue_filling ______________ self = def test_slow_priority_queue_filling(self): """ We discovered that adding tasks to a priority queue was a bottleneck when having a large (70k+ builds) queue, see #2095. Make sure this never happen again. """ tasks = [ToyQueueTask(i) for i in range(100000)] # We need to run this test with logging only INFO, otherwise we waste # around 5 seconds just on running self.log.debug because we need to # connect to redis for each call # The point of this test is to make sure that adding tasks to priority # queue is not a bottleneck on production, and we don't use DEBUG there # anyway. log.setLevel(logging.INFO) t1 = time.time() for task in tasks: self.worker_manager.add_task(task) t2 = time.time() # It should actually be faster than 1 second but I am adding one to # prevent false alarms in case somebody has a slow machine > assert t2 - t1 < 2 E assert (1704301544.4829688 - 1704301463.261207) < 2 tests/test_worker_manager.py:314: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 _____________ TestActionWorkerManager.test_run_starts_the_workers ______________ self = def test_run_starts_the_workers(self): self.worker_manager.run(timeout=0.0001) workers = self.workers() > assert len(workers) == 1 E assert 0 == 1 E + where 0 = len([]) tests/test_worker_manager.py:356: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704301547.4893014 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704301547.461983:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383124 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301547.461983:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.4221312999725342 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301547.461983:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383124 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704301623.657975 __________ TestActionWorkerManager.test_delete_not_allocated_workers ___________ self = def test_delete_not_allocated_workers(self): self.worker_manager.run(timeout=0.0001) > assert self.w0 in self.workers() E AssertionError: assert 'toy:1704301628.221708:0' in [] E + where 'toy:1704301628.221708:0' = .w0 E + and [] = >() E + where > = .workers tests/test_worker_manager.py:373: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704301628.2584426 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704301628.221708:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383133 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301628.221708:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.212555170059204 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301628.221708:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383133 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704301713.8255818 ___________ TestActionWorkerManager.test_delete_not_started_workers ____________ self = def test_delete_not_started_workers(self): self.worker_manager.environ = {'FAIL_EARLY': '1'} self.worker_manager.worker_timeout_start = 0 self.worker_manager.run(timeout=0.0001) > assert self.workers() == [self.w0] E AssertionError: assert [] == ['toy:1704301721.2901328:0'] E Right contains one more item: 'toy:1704301721.2901328:0' E Full diff: E - ['toy:1704301721.2901328:0'] E + [] tests/test_worker_manager.py:390: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704301721.6441498 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704301721.2901328:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383146 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301721.2901328:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.1038706302642822 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301721.2901328:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383146 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704301802.2287304 __ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED_PID] __ self = mc_time = fail = 'FAIL_STARTED_PID' @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED']) @patch('copr_common.worker_manager.time.time') def test_delete_not_finished_workers(self, mc_time, fail): self.worker_manager.environ = {fail: '1'} self.worker_manager.worker_timeout_deadcheck = 0.4 # each time.time() call incremented by 1 mc_time.side_effect = range(1000) # first loop just starts the toy:0 worker with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) params = self.wait_field(self.w0, 'started') assert self.w0 in self.workers() assert 'started' in params if fail == 'FAIL_STARTED': # make sure kernel cleans up the process, so the next wm.run() # certainly sets the 'delete' flag wait_pid_exit(params['PID']) # toy 0 is marked for deleting with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) > assert 'delete' in self.redis.hgetall(self.w0) E AssertionError: assert 'delete' in {} E + where {} = >>>('toy:1704301809.0328987:0') E + where >>> = Redis>>.hgetall E + where Redis>> = .redis E + and 'toy:1704301809.0328987:0' = .w0 tests/test_worker_manager.py:428: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 0 INFO root:worker_manager.py:482 Starting worker toy:1704301809.0328987:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383152 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301809.0328987:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301809.0328987:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383152 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 12 DEBUG root:worker_manager.py:425 Worker.run() start at time 14 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:553 checking worker toy:1704301809.0328987:0 ERROR root:worker_manager.py:557 dead worker toy:1704301809.0328987:0 INFO root:worker_manager.py:482 Starting worker toy:1704301809.0328987:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383161 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301809.0328987:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301809.0328987:1', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383161 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 30 ____ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED] ____ self = mc_time = , fail = 'FAIL_STARTED' @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED']) @patch('copr_common.worker_manager.time.time') def test_delete_not_finished_workers(self, mc_time, fail): self.worker_manager.environ = {fail: '1'} self.worker_manager.worker_timeout_deadcheck = 0.4 # each time.time() call incremented by 1 mc_time.side_effect = range(1000) # first loop just starts the toy:0 worker with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) params = self.wait_field(self.w0, 'started') assert self.w0 in self.workers() assert 'started' in params if fail == 'FAIL_STARTED': # make sure kernel cleans up the process, so the next wm.run() # certainly sets the 'delete' flag wait_pid_exit(params['PID']) # toy 0 is marked for deleting with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) > assert 'delete' in self.redis.hgetall(self.w0) E AssertionError: assert 'delete' in {} E + where {} = >>>('toy:1704301978.855289:0') E + where >>> = Redis>>.hgetall E + where Redis>> = .redis E + and 'toy:1704301978.855289:0' = .w0 tests/test_worker_manager.py:428: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 0 INFO root:worker_manager.py:482 Starting worker toy:1704301978.855289:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383168 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301978.855289:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301978.855289:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383168 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 12 DEBUG root:worker_manager.py:425 Worker.run() start at time 14 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:553 checking worker toy:1704301978.855289:0 ERROR root:worker_manager.py:557 dead worker toy:1704301978.855289:0 INFO root:worker_manager.py:482 Starting worker toy:1704301978.855289:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383175 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301978.855289:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301978.855289:1', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383175 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 30 ___________________ TestActionWorkerManager.test_all_passed ____________________ self = caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffb590b7a0> def test_all_passed(self, caplog): # It is a lot of fun with Popen(). It seems it has some zombie reaping # mechanism. If the calling function objects are destroyed (including # the Popen() return value reference), the future call to Popen() seems # to just reap the old Popen() processes. self.worker_manager.expected_terminations_in_cycle = 5 self.worker_manager.run(timeout=100) for i in range(0, 10): smsg = "Starting worker {}{}, task.priority=0" assert ('root', 20, smsg.format(self.wprefix, i)) in \ caplog.record_tuples > assert ('root', 20, 'Finished worker {}{}'.format(self.wprefix, i)) in caplog.record_tuples E assert ('root', 20, 'Finished worker toy:1704302128.9089105:0') in [('root', 10, 'Worker.run() start at time 1704302129.4638786'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704302128.9089105:0, task.priority=0'), ('root', 10, "background pid=1383187 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302128.9089105:0', '0'])"), ('root', 10, "starting-on-background-took 1.8483552932739258 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302128.9089105:0', '0'])"), ('root', 20, 'Starting worker toy:1704302128.9089105:1, task.priority=0'), ...] E + where [('root', 10, 'Worker.run() start at time 1704302129.4638786'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704302128.9089105:0, task.priority=0'), ('root', 10, "background pid=1383187 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302128.9089105:0', '0'])"), ('root', 10, "starting-on-background-took 1.8483552932739258 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302128.9089105:0', '0'])"), ('root', 20, 'Starting worker toy:1704302128.9089105:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffffb590b7a0>.record_tuples tests/test_worker_manager.py:449: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302129.4638786 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383187 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302128.9089105:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.8483552932739258 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302128.9089105:0', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383188 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704302128.9089105:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.177464485168457 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704302128.9089105:1', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:2, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383189 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704302128.9089105:2', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.3927643299102783 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704302128.9089105:2', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:3, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383190 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704302128.9089105:3', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 3.0204153060913086 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704302128.9089105:3', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:4, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383191 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704302128.9089105:4', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 3.1499881744384766 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704302128.9089105:4', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:0 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:5, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383193 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704302128.9089105:5', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2.795402765274048 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704302128.9089105:5', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:1 failed to start ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:2 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:6, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383194 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704302128.9089105:6', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2.181959390640259 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704302128.9089105:6', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:3 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:7, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383196 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1704302128.9089105:7', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.209120750427246 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1704302128.9089105:7', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:4 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:8, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383197 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1704302128.9089105:8', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.761000633239746 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1704302128.9089105:8', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704302128.9089105:9, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383198 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1704302128.9089105:9', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 3.1944260597229004 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1704302128.9089105:9', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:5 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:6 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:7 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:8 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302128.9089105:9 failed to start DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383188 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383187 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383190 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383191 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383189 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383197 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383193 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383198 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383194 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383196 DEBUG root:test_worker_manager.py:73 cleaned up 10, started 10 DEBUG root:worker_manager.py:475 Reaped 10 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302367.848775 _________ TestActionWorkerManager.test_empty_queue_but_workers_running _________ self = def test_empty_queue_but_workers_running(self): 'check that sleep(1) is done if queue is empty, but some workers exist' self.worker_manager.clean_tasks() # only one task, but it will take some time. self.worker_manager.task_sleep = 0.5 self.worker_manager.add_task(ToyQueueTask(0)) # start the worker self.worker_manager.run(timeout=0.0001) # start them task with patch('copr_common.worker_manager.time.sleep') as sleep: # we can spawn more workers, but queue is empty self.worker_manager.run(timeout=0.0001) assert sleep.called > assert len(self.worker_manager.worker_ids()) == 1 E assert 0 == 1 E + where 0 = len([]) E + where [] = >() E + where > = .worker_ids E + where = .worker_manager tests/test_worker_manager.py:477: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302443.3147225 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302443.2701135:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383242 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302443.2701135:0', '0.5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 0.4800589084625244 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302443.2701135:0', '0.5']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383242 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302513.7204711 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302514.4895353 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302514.888794 _____________ TestActionWorkerManager.test_that_we_check_aliveness _____________ self = def test_that_we_check_aliveness(self): """ Worker Manager checks whether worker is running each 'worker_timeout_deadcheck' period, check whether it works """ self.worker_manager.task_sleep = 3 # assure task takes some time self.worker_manager.clean_tasks() self.worker_manager.add_task(ToyQueueTask(0)) self.worker_manager.worker_timeout_start = 1 self.worker_manager.worker_timeout_deadcheck = 1.5 # start the worker self.worker_manager.run(timeout=0.0001) # let the task start self.wait_field(self.w0, 'PID') # timeout for liveness check not yet left self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) if 'checked' in params: # slow builder, the delay between previous two run() calls were so # long so the second one managed to even check whether the worker is # alive. So if that happened, the delay needs to be larger than # deadcheck at least. checked = float(params['checked']) started = float(params['allocated']) assert started + self.worker_manager.worker_timeout_deadcheck <= checked # time for check.. time.sleep(1.5) self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) > assert 'checked' in params E AssertionError: assert 'checked' in {} tests/test_worker_manager.py:522: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302519.945711 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302519.9135523:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383252 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302519.9135523:0', '3']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.2972791194915771 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302519.9135523:0', '3']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383252 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302580.9077632 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302600.4208953 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1704302519.9135523:0 DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302600.4516804 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302602.2149951 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302602.2280624 _____________ TestActionWorkerManager.test_max_workers_has_effect ______________ self = def test_max_workers_has_effect(self): self.worker_manager.max_workers = 1 self.worker_manager.run(timeout=1) > assert self.w0 in self.workers() E AssertionError: assert 'toy:1704302605.6688826:0' in [] E + where 'toy:1704302605.6688826:0' = .w0 E + and [] = >() E + where > = .workers tests/test_worker_manager.py:533: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302605.7008538 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302605.6688826:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383260 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302605.6688826:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 0.8446898460388184 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302605.6688826:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383260 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302660.2719004 =============================== warnings summary =============================== ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning) ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead override_loader = get_loader(override_package_name) ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead return find_loader(fullname) tests/test_frontend.py:14 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_frontend.py:29 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:18 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:24 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:21 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:26 SKIPPED (Fixme or remove, test doesn't work.) /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:31 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:36 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_action.py: 5 warnings tests/test_modifyrepo.py: 16 warnings tests/run/test_copr_prune_results.py: 3 warnings /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior. warnings.warn( tests/test_action.py::TestAction::test_comps_create /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). now = datetime.utcnow() tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14. rv = reductor(4) -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html =========================== short test summary info ============================ FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] = 13 failed, 163 passed, 11 skipped, 47 warnings, 1 error in 14733.55s (4:05:33) = ==================================== ERRORS ==================================== ___ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[True] ___ + Exception Group Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call | result: Optional[TResult] = func() | ^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in | lambda: ihook(item=item, **kwds), when=when, reraise=reraise | ^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__ | return self._hookexec(self.name, self._hookimpls, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec | return self._inner_hookexec(hook_name, methods, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall | return outcome.get_result() | ^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result | raise exc.with_traceback(exc.__traceback__) | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall | res = hook_impl.function(*args) | ^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown | item.session._setupstate.teardown_exact(nextitem) | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact | raise exceptions[0] | ExceptionGroup: errors while tearing down (2 sub-exceptions) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture | _call_with_optional_argument(func, method) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument | func(arg) | File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method | shutil.rmtree(self.workdir) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repon_ulge80' +---------------- 2 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory | shutil.rmtree(directory) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-6hvfzcao' +------------------------------------ __ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[False] ___ + Exception Group Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call | result: Optional[TResult] = func() | ^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in | lambda: ihook(item=item, **kwds), when=when, reraise=reraise | ^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__ | return self._hookexec(self.name, self._hookimpls, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec | return self._inner_hookexec(hook_name, methods, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall | return outcome.get_result() | ^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result | raise exc.with_traceback(exc.__traceback__) | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall | res = hook_impl.function(*args) | ^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown | item.session._setupstate.teardown_exact(nextitem) | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact | raise exceptions[0] | ExceptionGroup: errors while tearing down (2 sub-exceptions) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture | _call_with_optional_argument(func, method) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument | func(arg) | File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method | shutil.rmtree(self.workdir) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repoy16_kd9v' +---------------- 2 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory | shutil.rmtree(directory) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-jkr2nk3k' +------------------------------------ =================================== FAILURES =================================== __________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________ self = def test_copr_modifyrepo_locks(self): with _lock(self.workdir) as opts: cmd = [modifyrepo, opts.directory, '--log-to-stdout'] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: proc.communicate(timeout=2) assert 0 # this shouldn't happen except subprocess.TimeoutExpired: proc.kill() out, err = proc.communicate() assert b"acquired lock" not in out assert b"acquired lock" not in err # with released lock proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) > out, err = proc.communicate(timeout=5) tests/test_modifyrepo.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib64/python3.12/subprocess.py:1209: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.12/subprocess.py:2109: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = endtime = 2353053.97806015, orig_timeout = 5, stdout_seq = [], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-repo4lizes5f', '--log-to-stdout']' timed out after 5 seconds /usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:215 Lock 72057592818997616 acquired on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:247 Lock 72057592818997616 released on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock _____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________ self = f_second_build = run_bg = True @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. > assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) E AssertionError: assert False E + where False = call_copr_repo('/tmp/copr-backend-test-6hvfzcao/john/empty/fedora-rawhide-x86_64', add=['00000002-example']) tests/test_modifyrepo.py:617: AssertionError _____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________ self = f_second_build = run_bg = False @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. > assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) E AssertionError: assert False E + where False = call_copr_repo('/tmp/copr-backend-test-jkr2nk3k/john/empty/fedora-rawhide-x86_64', add=['00000002-example']) tests/test_modifyrepo.py:617: AssertionError ___________ TestLimitedWorkerManager.test_that_limits_are_respected ____________ self = mc_time = _mc_sleep = caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffb7569df0> @patch('copr_common.worker_manager.time.sleep') @patch('copr_common.worker_manager.time.time') def test_that_limits_are_respected(self, mc_time, _mc_sleep, caplog): # each time.time() call incremented by 1 self.worker_manager.task_sleep = 5 self.worker_manager.worker_timeout_start = 1000 mc_time.side_effect = range(1000) self.worker_manager.run(timeout=150) messages = [ "Task '4' skipped, limit info: 'even', " "matching: worker:0, worker:2", "Task '6' skipped, limit info: 'even', " "matching: worker:0, worker:2", "Task '7' skipped, limit info: 'odd', " "matching: worker:1, worker:3, worker:5", "Task '8' skipped, limit info: 'even', " "matching: worker:0, worker:2", "Task '9' skipped, limit info: 'odd', " "matching: worker:1, worker:3, worker:5", ] for msg in messages: assert ('root', logging.DEBUG, msg) in caplog.record_tuples # Even though the "even" limit kicked-out task 4, the task 5 is still # successfully started because that's the third "odd" task. The rest of # tasks is just skipped. assert ('root', logging.INFO, "Starting worker worker:5, task.priority=0") in \ caplog.record_tuples # finish the task now self.redis.hset("worker:5", "status", "0") self.setup_tasks() # re-calculate limits self.worker_manager.run(timeout=150) # check worker manager recognized the finished task > assert ('root', logging.INFO, "Finished worker worker:5") \ in caplog.record_tuples E assert ('root', 20, 'Finished worker worker:5') in [('root', 10, 'Worker.run() start at time 0'), ('root', 20, 'Starting worker worker:0, task.priority=0'), ('root', 10, "background pid=1383014 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, "starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker worker:1, task.priority=0'), ...] E + where [('root', 10, 'Worker.run() start at time 0'), ('root', 20, 'Starting worker worker:0, task.priority=0'), ('root', 10, "background pid=1383014 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, "starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5'])"), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker worker:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffffb7569df0>.record_tuples tests/test_worker_manager.py:241: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 0 INFO root:worker_manager.py:482 Starting worker worker:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383014 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383015 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:2, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383016 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:3, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383019 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:5, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383021 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383015 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383014 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383021 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383016 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383019 DEBUG root:test_worker_manager.py:73 cleaned up 5, started 5 DEBUG root:worker_manager.py:475 Reaped 5 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 162 DEBUG root:worker_manager.py:383 Task 0 already has a worker process DEBUG root:worker_manager.py:383 Task 1 already has a worker process DEBUG root:worker_manager.py:383 Task 2 already has a worker process DEBUG root:worker_manager.py:383 Task 3 already has a worker process DEBUG root:worker_manager.py:383 Task 3 already has a worker process DEBUG root:worker_manager.py:383 Task 3 already has a worker process DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:383 Task 5 already has a worker process DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 176 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:533 Finished worker worker:0 DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:525 Missing 'allocated' flag for worker worker:5 DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 332 ______________ TestWorkerManager.test_slow_priority_queue_filling ______________ self = def test_slow_priority_queue_filling(self): """ We discovered that adding tasks to a priority queue was a bottleneck when having a large (70k+ builds) queue, see #2095. Make sure this never happen again. """ tasks = [ToyQueueTask(i) for i in range(100000)] # We need to run this test with logging only INFO, otherwise we waste # around 5 seconds just on running self.log.debug because we need to # connect to redis for each call # The point of this test is to make sure that adding tasks to priority # queue is not a bottleneck on production, and we don't use DEBUG there # anyway. log.setLevel(logging.INFO) t1 = time.time() for task in tasks: self.worker_manager.add_task(task) t2 = time.time() # It should actually be faster than 1 second but I am adding one to # prevent false alarms in case somebody has a slow machine > assert t2 - t1 < 2 E assert (1704301487.021491 - 1704301406.468773) < 2 tests/test_worker_manager.py:314: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 _____________ TestActionWorkerManager.test_run_starts_the_workers ______________ self = def test_run_starts_the_workers(self): self.worker_manager.run(timeout=0.0001) workers = self.workers() > assert len(workers) == 1 E assert 0 == 1 E + where 0 = len([]) tests/test_worker_manager.py:356: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704301490.619384 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704301490.2575138:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383123 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301490.2575138:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 0.7075741291046143 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301490.2575138:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383123 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704301557.6445842 __________ TestActionWorkerManager.test_delete_not_allocated_workers ___________ self = def test_delete_not_allocated_workers(self): self.worker_manager.run(timeout=0.0001) > assert self.w0 in self.workers() E AssertionError: assert 'toy:1704301562.0314343:0' in [] E + where 'toy:1704301562.0314343:0' = .w0 E + and [] = >() E + where > = .workers tests/test_worker_manager.py:373: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704301562.6299782 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704301562.0314343:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383128 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301562.0314343:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.355708360671997 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301562.0314343:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383128 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704301632.1230707 ___________ TestActionWorkerManager.test_delete_not_started_workers ____________ self = def test_delete_not_started_workers(self): self.worker_manager.environ = {'FAIL_EARLY': '1'} self.worker_manager.worker_timeout_start = 0 self.worker_manager.run(timeout=0.0001) assert self.workers() == [self.w0] self.worker_manager.run(timeout=0.0001) # toy 0 is deleted now > assert self.workers() == [self.w1] E AssertionError: assert [] == ['toy:1704301636.8080275:1'] E Right contains one more item: 'toy:1704301636.8080275:1' E Full diff: [INFO/MainProcess] process shutting down [INFO/MainProcess] process shutting down [INFO/MainProcess] process shutting down [INFO/MainProcess] process shutting down E - ['toy:1704301636.8080275:1'] E + [] tests/test_worker_manager.py:393: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704301636.844993 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704301636.8080275:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383136 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301636.8080275:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.3986318111419678 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301636.8080275:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383136 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704301717.8512535 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704301717.880931 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704301636.8080275:0 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704301636.8080275:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383145 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301636.8080275:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.6979234218597412 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301636.8080275:1', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383145 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704301794.6587732 __ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED_PID] __ self = mc_time = fail = 'FAIL_STARTED_PID' @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED']) @patch('copr_common.worker_manager.time.time') def test_delete_not_finished_workers(self, mc_time, fail): self.worker_manager.environ = {fail: '1'} self.worker_manager.worker_timeout_deadcheck = 0.4 # each time.time() call incremented by 1 mc_time.side_effect = range(1000) # first loop just starts the toy:0 worker with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) params = self.wait_field(self.w0, 'started') assert self.w0 in self.workers() assert 'started' in params if fail == 'FAIL_STARTED': # make sure kernel cleans up the process, so the next wm.run() # certainly sets the 'delete' flag wait_pid_exit(params['PID']) # toy 0 is marked for deleting with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) > assert 'delete' in self.redis.hgetall(self.w0) E AssertionError: assert 'delete' in {} E + where {} = >>>('toy:1704301801.6677651:0') E + where >>> = Redis>>.hgetall E + where Redis>> = .redis E + and 'toy:1704301801.6677651:0' = .w0 tests/test_worker_manager.py:428: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 0 INFO root:worker_manager.py:482 Starting worker toy:1704301801.6677651:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383151 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301801.6677651:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301801.6677651:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383151 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 12 DEBUG root:worker_manager.py:425 Worker.run() start at time 14 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1704301801.6677651:0 INFO root:worker_manager.py:482 Starting worker toy:1704301801.6677651:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383160 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301801.6677651:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301801.6677651:1', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383160 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 29 ____ TestActionWorkerManager.test_delete_not_finished_workers[FAIL_STARTED] ____ self = mc_time = , fail = 'FAIL_STARTED' @pytest.mark.parametrize('fail', ['FAIL_STARTED_PID', 'FAIL_STARTED']) @patch('copr_common.worker_manager.time.time') def test_delete_not_finished_workers(self, mc_time, fail): self.worker_manager.environ = {fail: '1'} self.worker_manager.worker_timeout_deadcheck = 0.4 # each time.time() call incremented by 1 mc_time.side_effect = range(1000) # first loop just starts the toy:0 worker with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) params = self.wait_field(self.w0, 'started') assert self.w0 in self.workers() assert 'started' in params if fail == 'FAIL_STARTED': # make sure kernel cleans up the process, so the next wm.run() # certainly sets the 'delete' flag wait_pid_exit(params['PID']) # toy 0 is marked for deleting with patch('copr_common.worker_manager.time.sleep'): self.worker_manager.run(timeout=1) > assert 'delete' in self.redis.hgetall(self.w0) E AssertionError: assert 'delete' in {} E + where {} = >>>('toy:1704301974.2691727:0') E + where >>> = Redis>>.hgetall E + where Redis>> = .redis E + and 'toy:1704301974.2691727:0' = .w0 tests/test_worker_manager.py:428: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 0 INFO root:worker_manager.py:482 Starting worker toy:1704301974.2691727:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383166 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301974.2691727:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704301974.2691727:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383166 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 12 DEBUG root:worker_manager.py:425 Worker.run() start at time 14 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:525 Missing 'allocated' flag for worker toy:1704301974.2691727:0 INFO root:worker_manager.py:482 Starting worker toy:1704301974.2691727:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383174 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301974.2691727:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704301974.2691727:1', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383174 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 29 ___________________ TestActionWorkerManager.test_all_passed ____________________ self = caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffb58ff500> def test_all_passed(self, caplog): # It is a lot of fun with Popen(). It seems it has some zombie reaping # mechanism. If the calling function objects are destroyed (including # the Popen() return value reference), the future call to Popen() seems # to just reap the old Popen() processes. self.worker_manager.expected_terminations_in_cycle = 5 self.worker_manager.run(timeout=100) for i in range(0, 10): smsg = "Starting worker {}{}, task.priority=0" assert ('root', 20, smsg.format(self.wprefix, i)) in \ caplog.record_tuples > assert ('root', 20, 'Finished worker {}{}'.format(self.wprefix, i)) in caplog.record_tuples E assert ('root', 20, 'Finished worker toy:1704302123.2208183:0') in [('root', 10, 'Worker.run() start at time 1704302123.2544618'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704302123.2208183:0, task.priority=0'), ('root', 10, "background pid=1383181 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302123.2208183:0', '0'])"), ('root', 10, "starting-on-background-took 1.020233154296875 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302123.2208183:0', '0'])"), ('root', 20, 'Starting worker toy:1704302123.2208183:1, task.priority=0'), ...] E + where [('root', 10, 'Worker.run() start at time 1704302123.2544618'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704302123.2208183:0, task.priority=0'), ('root', 10, "background pid=1383181 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302123.2208183:0', '0'])"), ('root', 10, "starting-on-background-took 1.020233154296875 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302123.2208183:0', '0'])"), ('root', 20, 'Starting worker toy:1704302123.2208183:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffffb58ff500>.record_tuples tests/test_worker_manager.py:449: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302123.2544618 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302123.2208183:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383181 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302123.2208183:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.020233154296875 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302123.2208183:0', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704302123.2208183:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383183 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704302123.2208183:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 0.9469482898712158 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704302123.2208183:1', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704302123.2208183:2, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383184 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704302123.2208183:2', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.2531428337097168 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704302123.2208183:2', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302123.2208183:3, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383185 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704302123.2208183:3', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.476194143295288 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704302123.2208183:3', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704302123.2208183:4, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383186 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704302123.2208183:4', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.371199131011963 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704302123.2208183:4', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302123.2208183:4 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704302123.2208183:5, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383192 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704302123.2208183:5', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2.5758612155914307 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704302123.2208183:5', '0']) DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704302123.2208183:5 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704302123.2208183:6, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383199 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704302123.2208183:6', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 4.177871465682983 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704302123.2208183:6', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383181 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383184 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383183 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383186 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383185 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383192 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383199 DEBUG root:test_worker_manager.py:73 cleaned up 7, started 7 DEBUG root:worker_manager.py:475 Reaped 7 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302378.4561992 _________ TestActionWorkerManager.test_empty_queue_but_workers_running _________ self = def test_empty_queue_but_workers_running(self): 'check that sleep(1) is done if queue is empty, but some workers exist' self.worker_manager.clean_tasks() # only one task, but it will take some time. self.worker_manager.task_sleep = 0.5 self.worker_manager.add_task(ToyQueueTask(0)) # start the worker self.worker_manager.run(timeout=0.0001) # start them task with patch('copr_common.worker_manager.time.sleep') as sleep: # we can spawn more workers, but queue is empty self.worker_manager.run(timeout=0.0001) assert sleep.called > assert len(self.worker_manager.worker_ids()) == 1 E assert 0 == 1 E + where 0 = len([]) E + where [] = >() E + where > = .worker_ids E + where = .worker_manager tests/test_worker_manager.py:477: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302448.4946332 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302448.4452815:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383245 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302448.4452815:0', '0.5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.1430885791778564 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302448.4452815:0', '0.5']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383245 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302519.4594011 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302519.4887972 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302519.923575 _____________ TestActionWorkerManager.test_that_we_check_aliveness _____________ self = def test_that_we_check_aliveness(self): """ Worker Manager checks whether worker is running each 'worker_timeout_deadcheck' period, check whether it works """ self.worker_manager.task_sleep = 3 # assure task takes some time self.worker_manager.clean_tasks() self.worker_manager.add_task(ToyQueueTask(0)) self.worker_manager.worker_timeout_start = 1 self.worker_manager.worker_timeout_deadcheck = 1.5 # start the worker self.worker_manager.run(timeout=0.0001) # let the task start self.wait_field(self.w0, 'PID') # timeout for liveness check not yet left self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) if 'checked' in params: # slow builder, the delay between previous two run() calls were so # long so the second one managed to even check whether the worker is # alive. So if that happened, the delay needs to be larger than # deadcheck at least. checked = float(params['checked']) started = float(params['allocated']) assert started + self.worker_manager.worker_timeout_deadcheck <= checked # time for check.. time.sleep(1.5) self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) > assert 'checked' in params E AssertionError: assert 'checked' in {} tests/test_worker_manager.py:522: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302525.0763688 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302524.5377803:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383254 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302524.5377803:0', '3']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 0.8064842224121094 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302524.5377803:0', '3']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383254 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302584.4332616 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302601.6285138 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:553 checking worker toy:1704302524.5377803:0 DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302602.6644146 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302604.2382858 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:553 checking worker toy:1704302524.5377803:0 DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302605.6194253 _____________ TestActionWorkerManager.test_max_workers_has_effect ______________ self = def test_max_workers_has_effect(self): self.worker_manager.max_workers = 1 self.worker_manager.run(timeout=1) > assert self.w0 in self.workers() E AssertionError: assert 'toy:1704302608.8980782:0' in [] E + where 'toy:1704302608.8980782:0' = .w0 E + and [] = >() E + where > = .workers tests/test_worker_manager.py:533: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704302609.6141794 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704302608.8980782:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383261 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302608.8980782:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.0148370265960693 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704302608.8980782:0', '0']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383261 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704302664.5519917 =============================== warnings summary =============================== ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning) ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead override_loader = get_loader(override_package_name) ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead return find_loader(fullname) tests/test_frontend.py:14 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_frontend.py:29 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:18 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:24 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:21 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:26 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:31 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:36 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_action.py: 5 warnings tests/test_modifyrepo.py: 16 warnings tests/run/test_copr_prune_results.py: 3 warnings /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior. warnings.warn( tests/test_action.py::TestAction::test_comps_create /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). now = datetime.utcnow() tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14. rv = reductor(4) -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html =========================== short test summary info ============================ FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] = 14 failed, 162 passed, 11 skipped, 47 warnings, 2 errors in 14738.52s (4:05:38) = FAILED tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect PASSED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities PASSED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED tests/run/test_copr_prune_results.py::TestPruneResults::test_run SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_main SKIPPED (Fixme or remove, test doesn't work.) ==================================== ERRORS ==================================== ___ ERROR at teardown of TestModifyRepo.test_copr_repo_timeouted_check[True] ___ + Exception Group Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 341, in from_call | result: Optional[TResult] = func() | ^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 262, in | lambda: ihook(item=item, **kwds), when=when, reraise=reraise | ^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_hooks.py", line 433, in __call__ | return self._hookexec(self.name, self._hookimpls, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_manager.py", line 112, in _hookexec | return self._inner_hookexec(hook_name, methods, kwargs, firstresult) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 155, in _multicall | return outcome.get_result() | ^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/pluggy/_result.py", line 108, in get_result | raise exc.with_traceback(exc.__traceback__) | File "/usr/lib/python3.12/site-packages/pluggy/_callers.py", line 80, in _multicall | res = hook_impl.function(*args) | ^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 182, in pytest_runtest_teardown | item.session._setupstate.teardown_exact(nextitem) | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 537, in teardown_exact | raise exceptions[0] | ExceptionGroup: errors while tearing down (2 sub-exceptions) +-+---------------- 1 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 918, in xunit_setup_method_fixture | _call_with_optional_argument(func, method) | File "/usr/lib/python3.12/site-packages/_pytest/python.py", line 785, in _call_with_optional_argument | func(arg) | File "/builddir/build/BUILD/copr-backend-1.173/tests/test_modifyrepo.py", line 68, in teardown_method | shutil.rmtree(self.workdir) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-test-copr-repon_ulge80' +---------------- 2 ---------------- | Traceback (most recent call last): | File "/usr/lib/python3.12/site-packages/_pytest/runner.py", line 526, in teardown_exact | fin() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 686, in | subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1038, in finish | raise exc | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 1031, in finish | func() | File "/usr/lib/python3.12/site-packages/_pytest/fixtures.py", line 918, in _teardown_yield_fixture | next(it) | File "/builddir/build/BUILD/copr-backend-1.173/tests/conftest.py", line 28, in f_temp_directory | shutil.rmtree(directory) | File "/usr/lib64/python3.12/shutil.py", line 759, in rmtree | onexc(os.lstat, path, err) | File "/usr/lib64/python3.12/shutil.py", line 757, in rmtree | orig_st = os.lstat(path, dir_fd=dir_fd) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | FileNotFoundError: [Errno 2] No such file or directory: '/tmp/copr-backend-test-6hvfzcao' +------------------------------------ =================================== FAILURES =================================== __________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________ self = def test_copr_modifyrepo_locks(self): with _lock(self.workdir) as opts: cmd = [modifyrepo, opts.directory, '--log-to-stdout'] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: proc.communicate(timeout=2) assert 0 # this shouldn't happen except subprocess.TimeoutExpired: proc.kill() out, err = proc.communicate() assert b"acquired lock" not in out assert b"acquired lock" not in err # with released lock proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) > out, err = proc.communicate(timeout=5) tests/test_modifyrepo.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib64/python3.12/subprocess.py:1209: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.12/subprocess.py:2109: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = endtime = 2353053.97806015, orig_timeout = 5, stdout_seq = [], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-repo4lizes5f', '--log-to-stdout']' timed out after 5 seconds /usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:215 Lock 72057592818997616 acquired on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:247 Lock 72057592818997616 released on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock _____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________ self = f_second_build = run_bg = True @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. > assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) E AssertionError: assert False E + where False = call_copr_repo('/tmp/copr-backend-test-6hvfzcao/john/empty/fedora-rawhide-x86_64', add=['00000002-example']) tests/test_modifyrepo.py:617: AssertionError _____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________ self = f_second_build = run_bg = False @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) # sys.exit() can not be used in testsuite os._exit(0) # pylint: disable=protected-access with _lock(chrootdir): # give the child some time to fill its Redis keys sleeper = 1 while True: if len(self.redis.keys()) > 0: break sleeper += 1 time.sleep(0.1) > assert sleeper < 10*15 # 15s E assert 150 < (10 * 15) tests/test_modifyrepo.py:629: AssertionError ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592816599504 on /tmp/copr-test-copr-repoy16_kd9v/_@_tmp_@_copr-backend-test-jkr2nk3k_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:215 Lock 72057592816599504 acquired on /tmp/copr-test-copr-repoy16_kd9v/_@_tmp_@_copr-backend-test-jkr2nk3k_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592816599504 on /tmp/copr-test-copr-repoy16_kd9v/_@_tmp_@_copr-backend-test-jkr2nk3k_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:247 Lock 72057592816599504 released on /tmp/copr-test-copr-repoy16_kd9v/_@_tmp_@_copr-backend-test-jkr2nk3k_@_john_@_empty_@_fedora-rawhide-x86_64.lock ___________ TestLimitedWorkerManager.test_that_limits_are_respected ____________ self = mc_time = _mc_sleep = caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffb7542ae0> @patch('copr_common.worker_manager.time.sleep') @patch('copr_common.worker_manager.time.time') def test_that_limits_are_respected(self, mc_time, _mc_sleep, caplog): # each time.time() call incremented by 1 self.worker_manager.task_sleep = 5 self.worker_manager.worker_timeout_start = 1000 mc_time.side_effect = range(1000) > self.worker_manager.run(timeout=150) tests/test_worker_manager.py:211: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/test_worker_manager.py:83: in run return super().run(*args, **kwargs) /usr/lib/python3.12/site-packages/copr_common/worker_manager.py:475: in run self.log.debug("Reaped %s processes", self._clean_daemon_processes()) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = def _clean_daemon_processes(self): """ Check that we are not leaving any zombies behind us """ waited = super()._clean_daemon_processes() self.log.debug("cleaned up %s, started %s", waited, self.started_in_cycle) if waited != self.started_in_cycle: if self.expected_terminations_in_cycle is not None: assert self.expected_terminations_in_cycle == waited return waited > assert False E assert False tests/test_worker_manager.py:78: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 0 INFO root:worker_manager.py:482 Starting worker worker:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383006 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383007 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:2, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383009 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:3, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383010 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:5, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383011 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383006 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383011 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383007 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383009 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383010 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1382993 DEBUG root:test_worker_manager.py:73 cleaned up 6, started 5 ______________ TestWorkerManager.test_slow_priority_queue_filling ______________ self = def test_slow_priority_queue_filling(self): """ We discovered that adding tasks to a priority queue was a bottleneck when having a large (70k+ builds) queue, see #2095. Make sure this never happen again. """ tasks = [ToyQueueTask(i) for i in range(100000)] # We need to run this test with logging only INFO, otherwise we waste # around 5 seconds just on running self.log.debug because we need to # connect to redis for each call # The point of this test is to make sure that adding tasks to priority # queue is not a bottleneck on production, and we don't use DEBUG there # anyway. log.setLevel(logging.INFO) t1 = time.time() for task in tasks: self.worker_manager.add_task(task) t2 = time.time() # It should actually be faster than 1 second but I am adding one to # prevent false alarms in case somebody has a slow machine > assert t2 - t1 < 2 E assert (1704302993.74412 - 1704302920.6809506) < 2 tests/test_worker_manager.py:314: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ___________________ TestActionWorkerManager.test_all_passed ____________________ self = caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffb59061b0> def test_all_passed(self, caplog): # It is a lot of fun with Popen(). It seems it has some zombie reaping # mechanism. If the calling function objects are destroyed (including # the Popen() return value reference), the future call to Popen() seems # to just reap the old Popen() processes. self.worker_manager.expected_terminations_in_cycle = 5 self.worker_manager.run(timeout=100) for i in range(0, 10): smsg = "Starting worker {}{}, task.priority=0" assert ('root', 20, smsg.format(self.wprefix, i)) in \ caplog.record_tuples > assert ('root', 20, 'Finished worker {}{}'.format(self.wprefix, i)) in caplog.record_tuples E assert ('root', 20, 'Finished worker toy:1704303870.6867206:0') in [('root', 10, 'Worker.run() start at time 1704303871.2727954'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704303870.6867206:0, task.priority=0'), ('root', 10, "background pid=1383334 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704303870.6867206:0', '0'])"), ('root', 10, "starting-on-background-took 0.8680477142333984 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704303870.6867206:0', '0'])"), ('root', 20, 'Starting worker toy:1704303870.6867206:1, task.priority=0'), ...] E + where [('root', 10, 'Worker.run() start at time 1704303871.2727954'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704303870.6867206:0, task.priority=0'), ('root', 10, "background pid=1383334 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704303870.6867206:0', '0'])"), ('root', 10, "starting-on-background-took 0.8680477142333984 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704303870.6867206:0', '0'])"), ('root', 20, 'Starting worker toy:1704303870.6867206:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffffb59061b0>.record_tuples tests/test_worker_manager.py:449: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704303871.2727954 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383334 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704303870.6867206:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 0.8680477142333984 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704303870.6867206:0', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383335 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704303870.6867206:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2.3223280906677246 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704303870.6867206:1', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:2, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383336 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704303870.6867206:2', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.7946860790252686 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704303870.6867206:2', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:3, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383337 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704303870.6867206:3', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.8364064693450928 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704303870.6867206:3', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:4, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383338 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704303870.6867206:4', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.0506556034088135 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704303870.6867206:4', '0']) DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:1 failed to start ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:0 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:5, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383340 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704303870.6867206:5', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.0348472595214844 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704303870.6867206:5', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:6, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383341 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704303870.6867206:6', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.416820764541626 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704303870.6867206:6', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:3 failed to start ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:2 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:7, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383342 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1704303870.6867206:7', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 3.1965279579162598 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1704303870.6867206:7', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:4 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:8, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383343 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1704303870.6867206:8', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 3.149829864501953 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1704303870.6867206:8', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704303870.6867206:9, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383344 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1704303870.6867206:9', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 3.5475142002105713 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1704303870.6867206:9', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:5 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:6 failed to start ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:7 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:8 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704303870.6867206:9 failed to start DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383336 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383337 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383335 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383334 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383338 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383340 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383342 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383343 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383344 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383341 DEBUG root:test_worker_manager.py:73 cleaned up 10, started 10 DEBUG root:worker_manager.py:475 Reaped 10 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704304035.2591112 _________ TestActionWorkerManager.test_empty_queue_but_workers_running _________ self = def test_empty_queue_but_workers_running(self): 'check that sleep(1) is done if queue is empty, but some workers exist' self.worker_manager.clean_tasks() # only one task, but it will take some time. self.worker_manager.task_sleep = 0.5 self.worker_manager.add_task(ToyQueueTask(0)) # start the worker self.worker_manager.run(timeout=0.0001) # start them task with patch('copr_common.worker_manager.time.sleep') as sleep: # we can spawn more workers, but queue is empty self.worker_manager.run(timeout=0.0001) > assert sleep.called E AssertionError: assert False E + where False = .called tests/test_worker_manager.py:476: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704304114.5461488 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704304114.4804552:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383374 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704304114.4804552:0', '0.5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.2491331100463867 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704304114.4804552:0', '0.5']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383374 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704304179.728382 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704304180.0615137 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704304114.4804552:0 failed to start DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704304180.0995514 _____________ TestActionWorkerManager.test_that_we_check_aliveness _____________ self = def test_that_we_check_aliveness(self): """ Worker Manager checks whether worker is running each 'worker_timeout_deadcheck' period, check whether it works """ self.worker_manager.task_sleep = 3 # assure task takes some time self.worker_manager.clean_tasks() self.worker_manager.add_task(ToyQueueTask(0)) self.worker_manager.worker_timeout_start = 1 self.worker_manager.worker_timeout_deadcheck = 1.5 # start the worker self.worker_manager.run(timeout=0.0001) # let the task start self.wait_field(self.w0, 'PID') # timeout for liveness check not yet left self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) if 'checked' in params: # slow builder, the delay between previous two run() calls were so # long so the second one managed to even check whether the worker is # alive. So if that happened, the delay needs to be larger than # deadcheck at least. checked = float(params['checked']) started = float(params['allocated']) assert started + self.worker_manager.worker_timeout_deadcheck <= checked # time for check.. time.sleep(1.5) self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) > assert 'checked' in params E AssertionError: assert 'checked' in {} tests/test_worker_manager.py:522: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704304184.311999 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704304184.2547348:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383380 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704304184.2547348:0', '3']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.593285322189331 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704304184.2547348:0', '3']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383380 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704304241.8972974 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704304265.6596484 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:553 checking worker toy:1704304184.2547348:0 DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704304266.6887794 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704304268.2503412 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:533 Finished worker toy:1704304184.2547348:0 DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704304268.281444 =============================== warnings summary =============================== ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning) ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead override_loader = get_loader(override_package_name) ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead return find_loader(fullname) tests/test_frontend.py:14 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_frontend.py:29 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:18 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:24 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:21 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:26 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:31 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:36 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_action.py: 5 warnings tests/test_modifyrepo.py: 16 warnings tests/run/test_copr_prune_results.py: 3 warnings /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior. warnings.warn( tests/test_action.py::TestAction::test_comps_create /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). now = datetime.utcnow() tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14. rv = reductor(4) -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html =========================== short test summary info ============================ FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness ERROR tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] = 8 failed, 168 passed, 11 skipped, 47 warnings, 1 error in 16419.33s (4:33:39) = [INFO/MainProcess] process shutting down [INFO/MainProcess] process shutting down FAILED tests/test_worker_manager.py::TestWorkerManager::test_worker_starts PASSED tests/test_worker_manager.py::TestWorkerManager::test_number_of_tasks PASSED tests/test_worker_manager.py::TestWorkerManager::test_task_to_worker_id PASSED tests/test_worker_manager.py::TestWorkerManager::test_preexisting_broken_worker PASSED tests/test_worker_manager.py::TestWorkerManager::test_cancel_task PASSED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_run_starts_the_workers PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_allocated_workers PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_started_workers PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED_PID] PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_delete_not_finished_workers[FAIL_STARTED] PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker PASSED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_max_workers_has_effect PASSED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_actions_priorities PASSED tests/test_worker_manager.py::TestActionWorkerManagerPriorities::test_backend_priority_adjustments PASSED tests/run/test_copr_prune_results.py::TestPruneResults::test_run SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_project_skipped_when_acr_disabled SKIPPED (Fixme or remove, test doesn't work.) tests/run/test_copr_prune_results.py::TestPruneResults::test_main SKIPPED (Fixme or remove, test doesn't work.) =================================== FAILURES =================================== __________________ TestModifyRepo.test_copr_modifyrepo_locks ___________________ self = def test_copr_modifyrepo_locks(self): with _lock(self.workdir) as opts: cmd = [modifyrepo, opts.directory, '--log-to-stdout'] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: proc.communicate(timeout=2) assert 0 # this shouldn't happen except subprocess.TimeoutExpired: proc.kill() out, err = proc.communicate() assert b"acquired lock" not in out assert b"acquired lock" not in err # with released lock proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) > out, err = proc.communicate(timeout=5) tests/test_modifyrepo.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib64/python3.12/subprocess.py:1209: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.12/subprocess.py:2109: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = endtime = 2353053.97806015, orig_timeout = 5, stdout_seq = [], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['run/copr-repo', '/tmp/copr-test-copr-repo4lizes5f', '--log-to-stdout']' timed out after 5 seconds /usr/lib64/python3.12/subprocess.py:1253: TimeoutExpired ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:215 Lock 72057592818997616 acquired on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592818997616 on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock DEBUG filelock:_api.py:247 Lock 72057592818997616 released on /tmp/copr-test-copr-repo4lizes5f/_@_tmp_@_copr-test-copr-repo4lizes5f.lock _____________ TestModifyRepo.test_copr_repo_timeouted_check[True] ______________ self = f_second_build = run_bg = True @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) # sys.exit() can not be used in testsuite os._exit(0) # pylint: disable=protected-access with _lock(chrootdir): # give the child some time to fill its Redis keys sleeper = 1 while True: if len(self.redis.keys()) > 0: break sleeper += 1 time.sleep(0.1) > assert sleeper < 10*15 # 15s E assert 150 < (10 * 15) tests/test_modifyrepo.py:629: AssertionError ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592816893104 on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:215 Lock 72057592816893104 acquired on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592816893104 on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:247 Lock 72057592816893104 released on /tmp/copr-test-copr-repon_ulge80/_@_tmp_@_copr-backend-test-6hvfzcao_@_john_@_empty_@_fedora-rawhide-x86_64.lock _____________ TestModifyRepo.test_copr_repo_timeouted_check[False] _____________ self = f_second_build = run_bg = False @pytest.mark.parametrize('run_bg', [True, False]) @mock.patch.dict(os.environ, {'COPR_TESTSUITE_NO_OUTPUT': '1'}) def test_copr_repo_timeouted_check(self, f_second_build, run_bg): _unused = self ctx = f_second_build chroot = ctx.chroots[0] chrootdir = os.path.join(ctx.empty_dir, chroot) repodata = os.path.join(chrootdir, 'repodata') # empty repodata at the beginning empty_repodata = load_primary_xml(repodata) assert empty_repodata['names'] == set() pid = os.fork() if not pid: # give parent some time to lock the repo time.sleep(1) # Run the blocked (by parent) createrepo, it must finish soon # anway because parent will claim the task is done. assert call_copr_repo(chrootdir, add=[ctx.builds[1]]) # sys.exit() can not be used in testsuite os._exit(0) # pylint: disable=protected-access with _lock(chrootdir): # give the child some time to fill its Redis keys sleeper = 1 while True: if len(self.redis.keys()) > 0: break sleeper += 1 time.sleep(0.1) > assert sleeper < 10*15 # 15s E assert 150 < (10 * 15) tests/test_modifyrepo.py:629: AssertionError ------------------------------ Captured log call ------------------------------- DEBUG root:copr-repo:324 acquiring lock DEBUG filelock:_api.py:212 Attempting to acquire lock 72057592816599504 on /tmp/copr-test-copr-repomxlyevwj/_@_tmp_@_copr-backend-test-0jbratni_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:215 Lock 72057592816599504 acquired on /tmp/copr-test-copr-repomxlyevwj/_@_tmp_@_copr-backend-test-0jbratni_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG root:copr-repo:327 acquired lock DEBUG filelock:_api.py:244 Attempting to release lock 72057592816599504 on /tmp/copr-test-copr-repomxlyevwj/_@_tmp_@_copr-backend-test-0jbratni_@_john_@_empty_@_fedora-rawhide-x86_64.lock DEBUG filelock:_api.py:247 Lock 72057592816599504 released on /tmp/copr-test-copr-repomxlyevwj/_@_tmp_@_copr-backend-test-0jbratni_@_john_@_empty_@_fedora-rawhide-x86_64.lock ___________ TestLimitedWorkerManager.test_that_limits_are_respected ____________ self = mc_time = _mc_sleep = caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffb75572c0> @patch('copr_common.worker_manager.time.sleep') @patch('copr_common.worker_manager.time.time') def test_that_limits_are_respected(self, mc_time, _mc_sleep, caplog): # each time.time() call incremented by 1 self.worker_manager.task_sleep = 5 self.worker_manager.worker_timeout_start = 1000 mc_time.side_effect = range(1000) > self.worker_manager.run(timeout=150) tests/test_worker_manager.py:211: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tests/test_worker_manager.py:83: in run return super().run(*args, **kwargs) /usr/lib/python3.12/site-packages/copr_common/worker_manager.py:475: in run self.log.debug("Reaped %s processes", self._clean_daemon_processes()) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = def _clean_daemon_processes(self): """ Check that we are not leaving any zombies behind us """ waited = super()._clean_daemon_processes() self.log.debug("cleaned up %s, started %s", waited, self.started_in_cycle) if waited != self.started_in_cycle: if self.expected_terminations_in_cycle is not None: assert self.expected_terminations_in_cycle == waited return waited > assert False E assert False tests/test_worker_manager.py:78: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 0 INFO root:worker_manager.py:482 Starting worker worker:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383002 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'worker:0', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383003 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'worker:1', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:2, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383004 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'worker:2', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:3, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383005 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'worker:3', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '4' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker worker:5, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383008 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '5', 'worker:5', '5']) DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '6' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '7' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '8' skipped, limit info: 'even', matching: worker:0, worker:2 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:466 Task '9' skipped, limit info: 'odd', matching: worker:1, worker:3, worker:5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383003 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383002 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383005 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383004 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383008 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1382991 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1382825 DEBUG root:test_worker_manager.py:73 cleaned up 7, started 5 ______________ TestWorkerManager.test_slow_priority_queue_filling ______________ self = def test_slow_priority_queue_filling(self): """ We discovered that adding tasks to a priority queue was a bottleneck when having a large (70k+ builds) queue, see #2095. Make sure this never happen again. """ tasks = [ToyQueueTask(i) for i in range(100000)] # We need to run this test with logging only INFO, otherwise we waste # around 5 seconds just on running self.log.debug because we need to # connect to redis for each call # The point of this test is to make sure that adding tasks to priority # queue is not a bottleneck on production, and we don't use DEBUG there # anyway. log.setLevel(logging.INFO) t1 = time.time() for task in tasks: self.worker_manager.add_task(task) t2 = time.time() # It should actually be faster than 1 second but I am adding one to # prevent false alarms in case somebody has a slow machine > assert t2 - t1 < 2 E assert (1704304689.665234 - 1704304605.5020952) < 2 tests/test_worker_manager.py:314: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ___________________ TestActionWorkerManager.test_all_passed ____________________ self = caplog = <_pytest.logging.LogCaptureFixture object at 0xffffffb58b1280> def test_all_passed(self, caplog): # It is a lot of fun with Popen(). It seems it has some zombie reaping # mechanism. If the calling function objects are destroyed (including # the Popen() return value reference), the future call to Popen() seems # to just reap the old Popen() processes. self.worker_manager.expected_terminations_in_cycle = 5 self.worker_manager.run(timeout=100) for i in range(0, 10): smsg = "Starting worker {}{}, task.priority=0" assert ('root', 20, smsg.format(self.wprefix, i)) in \ caplog.record_tuples > assert ('root', 20, 'Finished worker {}{}'.format(self.wprefix, i)) in caplog.record_tuples E assert ('root', 20, 'Finished worker toy:1704305679.713139:0') in [('root', 10, 'Worker.run() start at time 1704305680.3205633'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704305679.713139:0, task.priority=0'), ('root', 10, "background pid=1383641 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305679.713139:0', '0'])"), ('root', 10, "starting-on-background-took 0.664811372756958 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305679.713139:0', '0'])"), ('root', 20, 'Starting worker toy:1704305679.713139:1, task.priority=0'), ...] E + where [('root', 10, 'Worker.run() start at time 1704305680.3205633'), ('root', 10, 'Trying to clean old workers'), ('root', 20, 'Starting worker toy:1704305679.713139:0, task.priority=0'), ('root', 10, "background pid=1383641 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305679.713139:0', '0'])"), ('root', 10, "starting-on-background-took 0.664811372756958 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305679.713139:0', '0'])"), ('root', 20, 'Starting worker toy:1704305679.713139:1, task.priority=0'), ...] = <_pytest.logging.LogCaptureFixture object at 0xffffffb58b1280>.record_tuples tests/test_worker_manager.py:449: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:425 Worker.run() start at time 1704305680.3205633 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383641 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305679.713139:0', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 0.664811372756958 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305679.713139:0', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:1, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383642 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704305679.713139:1', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.4648079872131348 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '2', '1', 'toy:1704305679.713139:1', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:2, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383643 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704305679.713139:2', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.0508320331573486 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '3', '2', 'toy:1704305679.713139:2', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:3, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383644 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704305679.713139:3', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.9408447742462158 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '4', '3', 'toy:1704305679.713139:3', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:4, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383645 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704305679.713139:4', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.3405048847198486 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '5', '4', 'toy:1704305679.713139:4', '0']) DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305679.713139:1 failed to start ERROR root:worker_manager.py:546 worker toy:1704305679.713139:0 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:5, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383647 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704305679.713139:5', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.3047089576721191 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '6', '5', 'toy:1704305679.713139:5', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:6, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383648 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704305679.713139:6', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.220871925354004 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '7', '6', 'toy:1704305679.713139:6', '0']) DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305679.713139:3 failed to start ERROR root:worker_manager.py:546 worker toy:1704305679.713139:2 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:7, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383649 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1704305679.713139:7', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.779627799987793 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '8', '7', 'toy:1704305679.713139:7', '0']) INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:8, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383650 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1704305679.713139:8', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 2.2216694355010986 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '9', '8', 'toy:1704305679.713139:8', '0']) DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305679.713139:4 failed to start INFO root:worker_manager.py:482 Starting worker toy:1704305679.713139:9, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383651 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1704305679.713139:9', '0']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.391897201538086 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '10', '9', 'toy:1704305679.713139:9', '0']) DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:443 Worker count on a limit 5 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305679.713139:6 failed to start ERROR root:worker_manager.py:546 worker toy:1704305679.713139:5 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305679.713139:7 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305679.713139:8 failed to start DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305679.713139:9 failed to start DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383644 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383642 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383641 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383643 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383645 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383647 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383648 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383649 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383651 DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383650 DEBUG root:test_worker_manager.py:73 cleaned up 10, started 10 DEBUG root:worker_manager.py:475 Reaped 10 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704305853.353009 _________ TestActionWorkerManager.test_empty_queue_but_workers_running _________ self = def test_empty_queue_but_workers_running(self): 'check that sleep(1) is done if queue is empty, but some workers exist' self.worker_manager.clean_tasks() # only one task, but it will take some time. self.worker_manager.task_sleep = 0.5 self.worker_manager.add_task(ToyQueueTask(0)) # start the worker self.worker_manager.run(timeout=0.0001) # start them task with patch('copr_common.worker_manager.time.sleep') as sleep: # we can spawn more workers, but queue is empty self.worker_manager.run(timeout=0.0001) > assert sleep.called E AssertionError: assert False E + where False = .called tests/test_worker_manager.py:476: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704305941.3626392 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704305940.897685:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383686 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305940.897685:0', '0.5']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.596775770187378 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704305940.897685:0', '0.5']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383686 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704306021.7096217 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704306021.7577438 DEBUG root:worker_manager.py:513 Trying to clean old workers ERROR root:worker_manager.py:546 worker toy:1704305940.897685:0 failed to start DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704306022.3014405 _____________ TestActionWorkerManager.test_that_we_check_aliveness _____________ self = def test_that_we_check_aliveness(self): """ Worker Manager checks whether worker is running each 'worker_timeout_deadcheck' period, check whether it works """ self.worker_manager.task_sleep = 3 # assure task takes some time self.worker_manager.clean_tasks() self.worker_manager.add_task(ToyQueueTask(0)) self.worker_manager.worker_timeout_start = 1 self.worker_manager.worker_timeout_deadcheck = 1.5 # start the worker self.worker_manager.run(timeout=0.0001) # let the task start self.wait_field(self.w0, 'PID') # timeout for liveness check not yet left self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) if 'checked' in params: # slow builder, the delay between previous two run() calls were so # long so the second one managed to even check whether the worker is # alive. So if that happened, the delay needs to be larger than # deadcheck at least. checked = float(params['checked']) started = float(params['allocated']) assert started + self.worker_manager.worker_timeout_deadcheck <= checked # time for check.. time.sleep(1.5) self.worker_manager.run(timeout=0.0001) params = self.redis.hgetall(self.w0) > assert 'checked' in params E AssertionError: assert 'checked' in {} tests/test_worker_manager.py:522: AssertionError ------------------------------ Captured log setup ------------------------------ DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 1 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 2 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 3 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 4 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 5 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 6 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 7 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 8 to queue, priority 0 DEBUG root:worker_manager.py:386 Adding task 9 to queue, priority 0 ------------------------------ Captured log call ------------------------------- DEBUG root:worker_manager.py:386 Adding task 0 to queue, priority 0 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704306028.9378266 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:482 Starting worker toy:1704306028.8888721:0, task.priority=0 DEBUG root:worker_manager.py:575 background pid=1383721 started (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704306028.8888721:0', '3']) DEBUG root:test_worker_manager.py:64 starting-on-background-took 1.3641388416290283 (['python3', '/builddir/build/BUILD/copr-backend-1.173/tests/action-processor.py', '1', '0', 'toy:1704306028.8888721:0', '3']) DEBUG root:worker_manager.py:590 Worker Manager waited for pid=1383721 DEBUG root:test_worker_manager.py:73 cleaned up 1, started 1 DEBUG root:worker_manager.py:475 Reaped 1 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704306108.1565883 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704306134.6995153 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:553 checking worker toy:1704306028.8888721:0 DEBUG root:worker_manager.py:454 No more tasks, waiting for workers DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704306136.2935183 DEBUG root:worker_manager.py:425 Worker.run() start at time 1704306137.9130785 DEBUG root:worker_manager.py:513 Trying to clean old workers INFO root:worker_manager.py:533 Finished worker toy:1704306028.8888721:0 DEBUG root:test_worker_manager.py:73 cleaned up 0, started 0 DEBUG root:worker_manager.py:475 Reaped 0 processes DEBUG root:worker_manager.py:476 Worker.run() stop at time 1704306137.9418695 =============================== warnings summary =============================== ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning) ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 ../../../../usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870 /usr/lib/python3.12/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 ../../../../usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108 /usr/lib/python3.12/site-packages/gi/overrides/__init__.py:108: DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead override_loader = get_loader(override_package_name) ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 ../../../../usr/lib64/python3.12/pkgutil.py:291 /usr/lib64/python3.12/pkgutil.py:291: DeprecationWarning: 'pkgutil.find_loader' is deprecated and slated for removal in Python 3.14; use importlib.util.find_spec() instead return find_loader(fullname) tests/test_frontend.py:14 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:14: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_frontend.py:29 /builddir/build/BUILD/copr-backend-1.173/tests/test_frontend.py:29: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:18 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:18: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/daemons/test_log.py:24 /builddir/build/BUILD/copr-backend-1.173/tests/daemons/test_log.py:24: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:21 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:21: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:26 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:26: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:31 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:31: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/run/test_copr_prune_results.py:36 /builddir/build/BUILD/copr-backend-1.173/tests/run/test_copr_prune_results.py:36: PytestDeprecationWarning: @pytest.yield_fixture is deprecated. Use @pytest.fixture instead; they are the same. @pytest.yield_fixture tests/test_action.py: 5 warnings tests/test_modifyrepo.py: 16 warnings tests/run/test_copr_prune_results.py: 3 warnings /usr/lib64/python3.12/tarfile.py:2220: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior. warnings.warn( tests/test_action.py::TestAction::test_comps_create /usr/lib/python3.12/site-packages/httpretty/core.py:1077: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). now = datetime.utcnow() tests/test_worker_manager.py::TestActionWorkerManager::test_add_task_for_running_worker /usr/lib64/python3.12/copy.py:151: DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14. rv = reductor(4) -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html =========================== short test summary info ============================ FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_modifyrepo_locks FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[True] FAILED tests/test_modifyrepo.py::TestModifyRepo::test_copr_repo_timeouted_check[False] FAILED tests/test_worker_manager.py::TestLimitedWorkerManager::test_that_limits_are_respected FAILED tests/test_worker_manager.py::TestWorkerManager::test_slow_priority_queue_filling FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_all_passed FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_empty_queue_but_workers_running FAILED tests/test_worker_manager.py::TestActionWorkerManager::test_that_we_check_aliveness ===== 8 failed, 168 passed, 11 skipped, 47 warnings in 18308.67s (5:05:08) ===== [INFO/MainProcess] process shutting down [INFO/MainProcess] process shutting down + cleanup + redis-cli -p 7777 shutdown + wait error: Bad exit status from /var/tmp/rpm-tmp.wYKJ8U (%check) RPM build errors: Bad exit status from /var/tmp/rpm-tmp.wYKJ8U (%check) Child return code was: 1 EXCEPTION: [Error('Command failed: \n # bash --login -c /usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec\n', 1)] Traceback (most recent call last): File "/usr/lib/python3.11/site-packages/mockbuild/trace_decorator.py", line 93, in trace result = func(*args, **kw) ^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/site-packages/mockbuild/util.py", line 597, in do_with_status raise exception.Error("Command failed: \n # %s\n%s" % (command, output), child.returncode) mockbuild.exception.Error: Command failed: # bash --login -c /usr/bin/rpmbuild -bb --noclean --target noarch --nodeps /builddir/build/SPECS/copr-backend.spec