From 36ad7b5a4ad53c9252fa1cefd0adbb0d1220bc44 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Tue, 13 Feb 2024 09:59:25 -0500 Subject: [PATCH 1/2] [state of supporting PR #772 as of the current Pull Request's creation] [_697] Github workflows for running PRC tests Co-authored-by: Terrell Russell Includes a docker compose configuration in which the PRC test suite can be run in a way representative of typical operational use, and the version of iRODS server and python interpreter that we install are also easily reconfigurable. Additionally, this affords us the opportunity to run the test suite on a client node all its own, the iRODS server being reachable only via the network. [_502] test harness and container-based tests A new test harness is introduced in which we construct a new container (using either Docker or podman) for each test program we run. This allows full customization of the container environment for the particular needs of each test. Accordingly, included with the Github workflows (in a separate commit) is a full run of the PRC test suite with the iRODS server and catalog DB server running in the same container as the client. In the process of putting old tests through new rigors, faults were found and corrected in some of those tests. completely remove {send,recv}_oneshot sync mechanism review comments Comments moved to *.yml files. README.md no longer needed. update irods/harness/README deindent correct forgotten comment symbol clearer comment in run_tests.sh suggestions nr. 1 10-22-25 13:19 suggestions nr. 2 10-22-25 smchg package repo chg comments chg --- .github/workflows/main.yml | 2 + .github/workflows/run-bats-tests.yml | 33 +++ .github/workflows/run-local-suite.yml | 31 +++ .github/workflows/run-the-tests.yml | 41 +++ Dockerfile.prc_test.centos | 1 - Dockerfile.prc_test.ubuntu | 1 - docker-testing/README.md | 47 ++++ .../harness-docker-compose-irods-4.yml | 1 + .../harness-docker-compose-irods-5.yml | 1 + docker-testing/harness-docker-compose.yml | 45 ++++ docker-testing/iinit.py | 48 ++++ docker-testing/irods_catalog_4/Dockerfile | 3 + .../irods_catalog_4/init-user-db.sh | 11 + docker-testing/irods_catalog_5/Dockerfile | 3 + .../irods_catalog_5/init-user-db.sh | 12 + .../irods_catalog_provider_4/Dockerfile | 55 ++++ .../irods_catalog_provider_4/entrypoint.sh | 49 ++++ .../setup-4.3.1.input | 28 ++ .../setup-4.3.2.input | 28 ++ .../setup-4.3.3.input | 28 ++ .../setup-4.3.4.input | 28 ++ .../irods_catalog_provider_5/Dockerfile | 59 +++++ .../irods_catalog_provider_5/entrypoint.sh | 50 ++++ .../setup-5.0.0.input | 26 ++ .../setup-5.0.1.input | 26 ++ .../setup-5.0.2.input | 26 ++ docker-testing/print_repo_root_location | 5 + docker-testing/python_client/Dockerfile | 3 + docker-testing/run_tests.sh | 45 ++++ docker-testing/start_containers.sh | 58 +++++ docker-testing/stop_containers.sh | 30 +++ irods/message/__init__.py | 2 +- irods/test/access_test.py | 6 +- irods/test/data_obj_test.py | 15 +- irods/test/exception_test.py | 4 +- .../test/harness/000_install-irods.Dockerfile | 14 + .../test/harness/001_bats-python3.Dockerfile | 5 + irods/test/harness/002_ssl-and-pam.Dockerfile | 1 + .../003_compile-specific-python.Dockerfile | 16 ++ irods/test/harness/README.md | 60 +++++ irods/test/harness/build-docker.sh | 39 +++ irods/test/harness/create_docker_images.sh | 13 + irods/test/harness/docker_container_driver.sh | 139 ++++++++++ irods/test/harness/install.sh | 159 ++++++++++++ irods/test/harness/install_python_rule_engine | 23 ++ .../harness/irods_version_greater_or_equal_to | 44 ++++ irods/test/harness/manage_irods5_procs | 57 ++++ irods/test/harness/most_recent_python.sh | 25 ++ irods/test/harness/print_repo_root_location | 5 + irods/test/harness/setup_python_rule_engine | 92 +++++++ .../harness/start_postgresql_and_irods.sh | 34 +++ irods/test/harness/test_script_parameters | 44 ++++ irods/test/login_auth_test.sh | 77 ++++++ irods/test/login_auth_test_1.py | 1 + irods/test/login_auth_test_2.py | 1 + .../test/login_auth_test_must_run_manually.py | 2 +- irods/test/meta_test.py | 11 +- irods/test/pam.bats/funcs | 108 -------- .../test001_pam_password_expiration.bats | 68 ----- ...pam_interactive_test_must_run_manually.py} | 0 irods/test/rule_test.py | 12 +- irods/test/runner.py | 67 ++++- irods/test/scripts/iinit.py | 1 + irods/test/scripts/run_suite_locally.sh | 42 +++ ...write_pam_credentials_to_secrets_file.bats | 8 +- ...c_write_irodsA_utility_in_native_mode.bats | 3 + ...ssue_362_rogue_chars_in_pam_password.bats} | 32 ++- irods/test/scripts/test_support_functions | 245 ++++++++++++++++++ irods/test/scripts/update_json_for_test | 69 +++++ irods/test/setupssl.py | 2 +- 70 files changed, 2142 insertions(+), 228 deletions(-) create mode 100644 .github/workflows/run-bats-tests.yml create mode 100644 .github/workflows/run-local-suite.yml create mode 100644 .github/workflows/run-the-tests.yml create mode 100644 docker-testing/README.md create mode 120000 docker-testing/harness-docker-compose-irods-4.yml create mode 120000 docker-testing/harness-docker-compose-irods-5.yml create mode 100644 docker-testing/harness-docker-compose.yml create mode 100755 docker-testing/iinit.py create mode 100644 docker-testing/irods_catalog_4/Dockerfile create mode 100644 docker-testing/irods_catalog_4/init-user-db.sh create mode 100644 docker-testing/irods_catalog_5/Dockerfile create mode 100644 docker-testing/irods_catalog_5/init-user-db.sh create mode 100644 docker-testing/irods_catalog_provider_4/Dockerfile create mode 100644 docker-testing/irods_catalog_provider_4/entrypoint.sh create mode 100644 docker-testing/irods_catalog_provider_4/setup-4.3.1.input create mode 100644 docker-testing/irods_catalog_provider_4/setup-4.3.2.input create mode 100644 docker-testing/irods_catalog_provider_4/setup-4.3.3.input create mode 100644 docker-testing/irods_catalog_provider_4/setup-4.3.4.input create mode 100644 docker-testing/irods_catalog_provider_5/Dockerfile create mode 100644 docker-testing/irods_catalog_provider_5/entrypoint.sh create mode 100644 docker-testing/irods_catalog_provider_5/setup-5.0.0.input create mode 100644 docker-testing/irods_catalog_provider_5/setup-5.0.1.input create mode 100644 docker-testing/irods_catalog_provider_5/setup-5.0.2.input create mode 100755 docker-testing/print_repo_root_location create mode 100644 docker-testing/python_client/Dockerfile create mode 100755 docker-testing/run_tests.sh create mode 100755 docker-testing/start_containers.sh create mode 100755 docker-testing/stop_containers.sh create mode 100644 irods/test/harness/000_install-irods.Dockerfile create mode 100644 irods/test/harness/001_bats-python3.Dockerfile create mode 100644 irods/test/harness/002_ssl-and-pam.Dockerfile create mode 100644 irods/test/harness/003_compile-specific-python.Dockerfile create mode 100644 irods/test/harness/README.md create mode 100755 irods/test/harness/build-docker.sh create mode 100755 irods/test/harness/create_docker_images.sh create mode 100755 irods/test/harness/docker_container_driver.sh create mode 100755 irods/test/harness/install.sh create mode 100755 irods/test/harness/install_python_rule_engine create mode 100755 irods/test/harness/irods_version_greater_or_equal_to create mode 100755 irods/test/harness/manage_irods5_procs create mode 100755 irods/test/harness/most_recent_python.sh create mode 100755 irods/test/harness/print_repo_root_location create mode 100755 irods/test/harness/setup_python_rule_engine create mode 100755 irods/test/harness/start_postgresql_and_irods.sh create mode 100644 irods/test/harness/test_script_parameters create mode 100755 irods/test/login_auth_test.sh create mode 120000 irods/test/login_auth_test_1.py create mode 120000 irods/test/login_auth_test_2.py delete mode 100644 irods/test/pam.bats/funcs delete mode 100644 irods/test/pam.bats/test001_pam_password_expiration.bats rename irods/test/{pam_interactive_test.py => pam_interactive_test_must_run_manually.py} (100%) create mode 120000 irods/test/scripts/iinit.py create mode 100755 irods/test/scripts/run_suite_locally.sh rename irods/test/{PRC_issue_362.bats => scripts/test010_issue_362_rogue_chars_in_pam_password.bats} (64%) mode change 100644 => 100755 create mode 100644 irods/test/scripts/test_support_functions create mode 100644 irods/test/scripts/update_json_for_test diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40ad105d3..1c9fcf912 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,8 @@ # https://github.com/actions/starter-workflows/blob/master/ci/python-package.yml # (C) Github, MIT License +# Static type checking tests using `mypy`. + name: "Python type checking" on: [push, pull_request] diff --git a/.github/workflows/run-bats-tests.yml b/.github/workflows/run-bats-tests.yml new file mode 100644 index 000000000..c79d79291 --- /dev/null +++ b/.github/workflows/run-bats-tests.yml @@ -0,0 +1,33 @@ +# Run a set of tests, each in its own container and with a potentially customized setup. +# (Documentation and implementation for the test harness may be found in `irods/test/harness`.) + +name: run-bats-tests + +on: [push, pull_request] + +jobs: + tests: + timeout-minutes: 20 + + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./irods/test/harness + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + for script in ../scripts/test[0-9]*; do + ./docker_container_driver.sh -V $script + done diff --git a/.github/workflows/run-local-suite.yml b/.github/workflows/run-local-suite.yml new file mode 100644 index 000000000..85bd810ac --- /dev/null +++ b/.github/workflows/run-local-suite.yml @@ -0,0 +1,31 @@ +# Run the client test suite in a Docker container, targeting a locally running instance of the iRODS server. +# (Documentation and implementation for the test harness may be found in `irods/test/harness`.) + +name: run-local-suite + +on: [push, pull_request] + +jobs: + tests: + timeout-minutes: 20 + + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./irods/test/harness + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + ./docker_container_driver.sh -V ../scripts/run_suite_locally.sh diff --git a/.github/workflows/run-the-tests.yml b/.github/workflows/run-the-tests.yml new file mode 100644 index 000000000..7d1c44784 --- /dev/null +++ b/.github/workflows/run-the-tests.yml @@ -0,0 +1,41 @@ +# Create a networked set of containers (via a Docker compose project) on which to run the client test suite. +# (For further information, see the README in `docker-testing`.) + +name: run-the-tests + +on: [push, pull_request] + +jobs: + tests: + timeout-minutes: 20 + + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./docker-testing + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start containers + run: ./start_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run test + run: | + while :; do + client_container=$(docker ps --format "{{.Names}}"|grep python.client) + [ -n "$client_container" ] && break + sleep 1 + done + echo "client_container = [$client_container]" + docker exec "${client_container}" /repo_root/docker-testing/run_tests.sh + + - name: Stop containers + if: always() + run: ./stop_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" diff --git a/Dockerfile.prc_test.centos b/Dockerfile.prc_test.centos index debed6d6d..c171fa62e 100644 --- a/Dockerfile.prc_test.centos +++ b/Dockerfile.prc_test.centos @@ -24,6 +24,5 @@ RUN python${py_N} repo/docker_build/iinit.py \ password rods SHELL ["/bin/bash","-c"] CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/Dockerfile.prc_test.ubuntu b/Dockerfile.prc_test.ubuntu index e8c958a85..79ed07e12 100644 --- a/Dockerfile.prc_test.ubuntu +++ b/Dockerfile.prc_test.ubuntu @@ -31,6 +31,5 @@ SHELL ["/bin/bash","-c"] # 3. run python tests as the new group CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/docker-testing/README.md b/docker-testing/README.md new file mode 100644 index 000000000..980c9c2bf --- /dev/null +++ b/docker-testing/README.md @@ -0,0 +1,47 @@ +# A Topological Setup for Testing the Python Client + +The `docker-testing` directory contains the necessary files for building and +running tests from the perspective of a specific client node in a larger network. + +We currently allow a choice of Python interpreter and iRODS server to be installed +on the client and provider nodes of a simulated network topology. + +The choice of versions are dictated when running the test: + +|:------------------:|:---------------:| +|Environment Variable| Valid Range | +|:-------------------|-----------------| +IRODS_PACKAGE_VERSION|4.3.1 to 5.0.2 | +PYTHON_VERSION |3.9 to 3.13 | +|:-------------------|-----------------| + +Currently the database server is fixed as Postgres. + +## Details of usage + +The file `$REPO/.github/workflows/run-the-tests.yml` +(where `$REPO` is the `/path/to/local/python-irodsclient` repository) +contains commands for starting the server and client containers and running the PRC +suite in response to a push or pull-request. + +The test suite can also be run on any workstation with docker compose installed. +What follows is a short summary of how to run the test configuration at the bench. +It is this procedure which is run within the Github workflows. + + 1. cd into top level of $REPO + + 2. run: + ``` + ./docker-testing/start_containers.sh 4.3.4 3.11 + ``` + This builds and runs the docker images for the project, with "4.3.4" being the iRODS + version installed on the provider and "3.11" is the version of python run on the client side. + + 3. run: + ``` + docker exec /repo_root/docker-testing/run_tests.sh + ``` + (Note: `/repo_root` is an actual literal path, internal to the container.) + You'll see the test output displayed on the console. At completion, xmlrunner outputs are in /tmp. + + 4. use `docker logs -f` with the provider instance name to tail the irods server log output diff --git a/docker-testing/harness-docker-compose-irods-4.yml b/docker-testing/harness-docker-compose-irods-4.yml new file mode 120000 index 000000000..db32ad86f --- /dev/null +++ b/docker-testing/harness-docker-compose-irods-4.yml @@ -0,0 +1 @@ +harness-docker-compose.yml \ No newline at end of file diff --git a/docker-testing/harness-docker-compose-irods-5.yml b/docker-testing/harness-docker-compose-irods-5.yml new file mode 120000 index 000000000..db32ad86f --- /dev/null +++ b/docker-testing/harness-docker-compose-irods-5.yml @@ -0,0 +1 @@ +harness-docker-compose.yml \ No newline at end of file diff --git a/docker-testing/harness-docker-compose.yml b/docker-testing/harness-docker-compose.yml new file mode 100644 index 000000000..abeb44d3c --- /dev/null +++ b/docker-testing/harness-docker-compose.yml @@ -0,0 +1,45 @@ +version: '3' + +services: + irods-catalog: + build: + context: irods_catalog_${irods_major} + # 5432 is exposed by default and can conflict with other postgres containers. + # When the metalnx-db service is no longer needed, this stanza can be removed. + ports: + - "5430:5432" + environment: + - POSTGRES_PASSWORD=testpassword + + python-client: + build: + context: python_client + args: + python_version: ${python_version} + command: + tail -f /dev/null + volumes: + - ${repo_external}:/repo_root:ro + - /tmp/irods-client-share.py-${python_version}:/irods_shared + depends_on: + irods-catalog-provider: + condition: service_healthy + + irods-catalog-provider: + volumes: + - /tmp/irods-client-share.py-${python_version}:/irods_shared + build: + context: irods_catalog_provider_${irods_major} + args: + irods_version: ${irods_version} + shm_size: 500mb + healthcheck: + test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + ports: + - "1247:1247" + depends_on: + - irods-catalog + diff --git a/docker-testing/iinit.py b/docker-testing/iinit.py new file mode 100755 index 000000000..776ca97ee --- /dev/null +++ b/docker-testing/iinit.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# This script creates the client environment to authenticate natively +# as the 'rods' admin on the Docker node running the Python client tests. +# Thus, we don't need irods-icommands to be installed on that node. + +from irods.password_obfuscation import encode +import json +import os +import sys +from os import chmod +from os.path import expanduser,exists,join +from getopt import getopt + + +home_env_path = expanduser('~/.irods') +env_file_path = join(home_env_path,'irods_environment.json') +auth_file_path = join(home_env_path,'.irodsA') + + +def do_iinit(host, port, user, zone, password): + if not exists(home_env_path): + os.makedirs(home_env_path) + else: + raise RuntimeError('~/.irods already exists') + + with open(env_file_path,'w') as env_file: + json.dump ( { "irods_host": host, + "irods_port": int(port), + "irods_user_name": user, + "irods_zone_name": zone }, env_file, indent=4) + with open(auth_file_path,'w') as auth_file: + auth_file.write(encode(password)) + chmod (auth_file_path,0o600) + + +def get_kv_pairs_from_cmdline(*args): + arglist = list(args) + while arglist: + k = arglist.pop(0) + v = arglist.pop(0) + yield k,v + + +if __name__ == '__main__': + args = sys.argv[1:] + dct = {k:v for k,v in get_kv_pairs_from_cmdline(*args)} + do_iinit(**dct) diff --git a/docker-testing/irods_catalog_4/Dockerfile b/docker-testing/irods_catalog_4/Dockerfile new file mode 100644 index 000000000..f02c4520c --- /dev/null +++ b/docker-testing/irods_catalog_4/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:12 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/docker-testing/irods_catalog_4/init-user-db.sh b/docker-testing/irods_catalog_4/init-user-db.sh new file mode 100644 index 000000000..5ff6b0375 --- /dev/null +++ b/docker-testing/irods_catalog_4/init-user-db.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; +EOSQL diff --git a/docker-testing/irods_catalog_5/Dockerfile b/docker-testing/irods_catalog_5/Dockerfile new file mode 100644 index 000000000..112ffbaaa --- /dev/null +++ b/docker-testing/irods_catalog_5/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:16 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/docker-testing/irods_catalog_5/init-user-db.sh b/docker-testing/irods_catalog_5/init-user-db.sh new file mode 100644 index 000000000..f3c724e2f --- /dev/null +++ b/docker-testing/irods_catalog_5/init-user-db.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; + ALTER DATABASE "ICAT" OWNER TO irods +EOSQL diff --git a/docker-testing/irods_catalog_provider_4/Dockerfile b/docker-testing/irods_catalog_provider_4/Dockerfile new file mode 100644 index 000000000..7632b273f --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/Dockerfile @@ -0,0 +1,55 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ focal main" | tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ARG irods_version=4.3.1 +ARG irods_package_version_suffix=-0~focal +ARG irods_package_version=${irods_version}${irods_package_version_suffix} +ARG irods_resource_plugin_version=${irods_version}.0${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +RUN chmod u+x ./entrypoint.sh +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker-testing/irods_catalog_provider_4/entrypoint.sh b/docker-testing/irods_catalog_provider_4/entrypoint.sh new file mode 100644 index 000000000..3a321db2f --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/entrypoint.sh @@ -0,0 +1,49 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q +do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py < "${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared + +#TODO ensure this is done for 4.3+ only. 4.2 doesn't have this server config key +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && \ + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && \ + touch ~/provider-address.do_not_remove + } +} || { echo >&2 "Error modifying $ORIG_SERVER_CONFIG"; exit 1; } + +echo "Starting server" + +cd /usr/sbin +su irods -c 'bash -c "./irodsServer -u"' + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.1.input b/docker-testing/irods_catalog_provider_4/setup-4.3.1.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.1.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.2.input b/docker-testing/irods_catalog_provider_4/setup-4.3.2.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.2.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.3.input b/docker-testing/irods_catalog_provider_4/setup-4.3.3.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.3.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.4.input b/docker-testing/irods_catalog_provider_4/setup-4.3.4.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.4.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_5/Dockerfile b/docker-testing/irods_catalog_provider_5/Dockerfile new file mode 100644 index 000000000..58daffd87 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/Dockerfile @@ -0,0 +1,59 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ noble main" | tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + postgresql-client-16 \ + vim-tiny \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +# - postgres client 16 is for pg_isready +# TODO delete vim-tiny + +ARG irods_version=5.0.1 +ARG irods_package_version_suffix=-0~noble +ARG irods_package_version=${irods_version}${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +RUN chmod u+x ./entrypoint.sh +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker-testing/irods_catalog_provider_5/entrypoint.sh b/docker-testing/irods_catalog_provider_5/entrypoint.sh new file mode 100644 index 000000000..b3aec63cf --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/entrypoint.sh @@ -0,0 +1,50 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q +do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py < "${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared +chmod 0777 /irods_shared + +#TODO ensure this is done for 4.3+ only. 4.2 doesn't have this server config key +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && \ + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && \ + touch ~/provider-address.do_not_remove + } +} || { echo >&2 "Error modifying $ORIG_SERVER_CONFIG"; exit 1; } + +echo "Starting server" + +cd /usr/sbin +su irods -c 'bash -c "./irodsServer -p /tmp/irods.pid"' + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.0.input b/docker-testing/irods_catalog_provider_5/setup-5.0.0.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.0.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.1.input b/docker-testing/irods_catalog_provider_5/setup-5.0.1.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.1.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.2.input b/docker-testing/irods_catalog_provider_5/setup-5.0.2.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.2.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/print_repo_root_location b/docker-testing/print_repo_root_location new file mode 100755 index 000000000..79d91af7e --- /dev/null +++ b/docker-testing/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/docker-testing/python_client/Dockerfile b/docker-testing/python_client/Dockerfile new file mode 100644 index 000000000..83c7ff885 --- /dev/null +++ b/docker-testing/python_client/Dockerfile @@ -0,0 +1,3 @@ +ARG python_version +FROM python:${python_version} +RUN pip install remote-pdb diff --git a/docker-testing/run_tests.sh b/docker-testing/run_tests.sh new file mode 100755 index 000000000..8ed4a77af --- /dev/null +++ b/docker-testing/run_tests.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e -x +PYTHON=$(which python3) +if [ -z "$PYTHON" ]; then + PYTHON=$(which python) +fi +DIR=$(dirname "$0") +cd "$DIR" + +REPO="$(./print_repo_root_location)" + +if [ -d /irods_shared ]; then + + # Get the numeric user and group id's for irods service account on the provider. This helps to set up the test user + # (named 'user') with proper permissions for the shared volume on the client node. + groupadd -o -g $(stat -c%g /irods_shared) irods + useradd -g irods -u $(stat -c%u /irods_shared) irods + + # Set up useful subdirectories in the client/provider shared volume. + mkdir /irods_shared/{tmp,reg_resc} + chown irods:irods /irods_shared/{tmp,reg_resc} + chmod 777 /irods_shared/reg_resc + chmod g+ws /irods_shared/tmp + + # Make a test user in group irods, who will run the client tests. + useradd -G irods -m -s/bin/bash user + + # Create writable copy of this repo. + cp -r /"$REPO"{,.copy} + REPO+=.copy + chown -R user "$REPO" + chmod u+w "$REPO"/irods/test/test-data + + # Install PRC from the repo. + $PYTHON -m pip install "$REPO[tests]" +fi + +su - user -c "\ +$PYTHON '$DIR'/iinit.py \ + host irods-catalog-provider \ + port 1247 \ + user rods \ + password rods \ + zone tempZone +$PYTHON '$REPO'/irods/test/runner.py $*" diff --git a/docker-testing/start_containers.sh b/docker-testing/start_containers.sh new file mode 100755 index 000000000..b4b748cb3 --- /dev/null +++ b/docker-testing/start_containers.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 [-n] [-b ""] [irods_version] python_version"; exit 2; +} + +SHELL_DOCKER_COMPOSE_BUILD_ARGS="" +DO_NOT_RUN="" + +while [[ $1 = -* ]]; do + if [ "$1" = "-b" ]; then + SHELL_DOCKER_COMPOSE_BUILD_ARGS=$2 + shift 2 + fi + if [ "$1" = "-n" ]; then + DO_NOT_RUN=1 + shift + fi +done + +if [ $# -eq 2 ]; then + IRODS_VERSION=$1 + PYTHON_VERSION=$2 +elif [ $# -eq 1 ]; then + IRODS_VERSION=4.3.4 + PYTHON_VERSION=$1 +else + usage +fi + +shift $# + +[ -n "$PYTHON_VERSION" -a -n "$IRODS_VERSION" ] || { + usage +} + +IRODS_MAJOR=${IRODS_VERSION//.*/} + +DIR=$(dirname "$0") +cd "${DIR}" +REPO_ROOT=$(realpath ..) + +echo "\ +repo_external=\"${REPO_ROOT}\" +python_version=\"${PYTHON_VERSION}\" +irods_version=\"${IRODS_VERSION}\" +irods_major=\"${IRODS_MAJOR}\"" >.env + +# In case the docker-compose setup varies between iRODS major releases, the .YML file may be a symbolic link. + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml build $SHELL_DOCKER_COMPOSE_BUILD_ARGS + +if [ -z "$DO_NOT_RUN" ]; then + docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml up -d +fi diff --git a/docker-testing/stop_containers.sh b/docker-testing/stop_containers.sh new file mode 100755 index 000000000..c41002287 --- /dev/null +++ b/docker-testing/stop_containers.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 [irods_version] python_version"; exit 1; +} + +if [ $# -eq 2 ]; then + IRODS_VERSION=$1 + PYTHON_VERSION=$2 +elif [ $# -eq 1 ]; then + IRODS_VERSION=4.3.4 + PYTHON_VERSION=$1 +else + usage +fi + +shift $# + +[ -n "$PYTHON_VERSION" -a -n "$IRODS_VERSION" ] || { + usage +} + +IRODS_MAJOR=${IRODS_VERSION//.*/} + +# In case the docker-compose setup varies between iRODS major releases, the .YML file may be a symbolic link. + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml down diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 818ae3677..9b8e0ec80 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -181,7 +181,7 @@ def ET(xml_type=(), server_version=None): logger = logging.getLogger(__name__) -IRODS_VERSION = (5, 0, 1, "d") +IRODS_VERSION = (5, 0, 2, "d") UNICODE = str diff --git a/irods/test/access_test.py b/irods/test/access_test.py index 7c496c685..73c122b76 100644 --- a/irods/test/access_test.py +++ b/irods/test/access_test.py @@ -533,11 +533,7 @@ def test_iRODSAccess_cannot_be_constructed_using_unsupported_type__issue_558(sel # Before the fix in #558, this would have been allowed and only later would the type discrepancy be revealed, # leading to opaque error messages. Now, the types are checked on the way in to ensure clarity and correctness. # TODO(#480): We cannot use the unittest.assertRaises context manager as this was introduced in python 3.1. - assertCall = getattr(self, "assertRaisesRegex", None) - if assertCall is None: - assertCall = self.assertRaisesRegexp - - assertCall( + self.assertRaisesRegex( TypeError, "'path' parameter must be of type 'str', 'irods.collection.iRODSCollection', " "'irods.data_object.iRODSDataObject', or 'irods.path.iRODSPath'.", diff --git a/irods/test/data_obj_test.py b/irods/test/data_obj_test.py index b6ed95886..6159a220a 100644 --- a/irods/test/data_obj_test.py +++ b/irods/test/data_obj_test.py @@ -3305,16 +3305,13 @@ def test_access_time__issue_700(self): if self.sess.server_version < (5,): self.skipTest("iRODS servers < 5.0.0 do not provide an access_time attribute for data objects.") - data_path= iRODSPath(self.coll.path, - unique_name(my_function_name(), datetime.now()) - ) - with self.sess.data_objects.open(data_path,"w") as f: - f.write(b'_') - with self.sess.data_objects.open(data_path,"r") as f: - f.read() + # Create a new, uniquely named test data object. + data = self.sess.data_objects.create( + f'{helpers.home_collection(self.sess)}/{unique_name(my_function_name(), datetime.now())}' + ) - data = self.sess.data_objects.get(data_path) - self.assertGreaterEqual(data.access_time, data.modify_time) + # Test that access_time is there, and of the right type. + self.assertIs(type(data.access_time), datetime) if __name__ == "__main__": # let the tests find the parent irods lib diff --git a/irods/test/exception_test.py b/irods/test/exception_test.py index 16a95c989..b400cf2aa 100644 --- a/irods/test/exception_test.py +++ b/irods/test/exception_test.py @@ -41,8 +41,8 @@ def test_400(self): excep_repr = repr(exc) errno_object = irods.exception.Errno(errno.EACCES) errno_repr = repr(errno_object) - self.assertRegexpMatches(errno_repr, r"\bErrno\b") - self.assertRegexpMatches( + self.assertRegex(errno_repr, r"\bErrno\b") + self.assertRegex( errno_repr, """['"]{msg}['"]""".format(msg=os.strerror(errno.EACCES)) ) self.assertIn(errno_repr, excep_repr) diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile new file mode 100644 index 000000000..b147aab4e --- /dev/null +++ b/irods/test/harness/000_install-irods.Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:22.04 +COPY install.sh / +ARG irods_package_version +ENV IRODS_PACKAGE_VERSION "$irods_package_version" +RUN for phase in initialize install-essential-packages add-package-repo; do \ + bash /install.sh --w=$phase 0; \ + done +RUN /install.sh 4 +COPY start_postgresql_and_irods.sh manage_irods5_procs / +RUN apt install -y sudo +RUN useradd -ms/bin/bash testuser +RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers +RUN apt install -y faketime +CMD bash /start_postgresql_and_irods.sh diff --git a/irods/test/harness/001_bats-python3.Dockerfile b/irods/test/harness/001_bats-python3.Dockerfile new file mode 100644 index 000000000..b78179856 --- /dev/null +++ b/irods/test/harness/001_bats-python3.Dockerfile @@ -0,0 +1,5 @@ +FROM install-irods +RUN apt update; apt install -y python3-pip bats +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install virtualenv +RUN python3 -m virtualenv /py3 diff --git a/irods/test/harness/002_ssl-and-pam.Dockerfile b/irods/test/harness/002_ssl-and-pam.Dockerfile new file mode 100644 index 000000000..810d10e86 --- /dev/null +++ b/irods/test/harness/002_ssl-and-pam.Dockerfile @@ -0,0 +1 @@ +FROM bats-python3 diff --git a/irods/test/harness/003_compile-specific-python.Dockerfile b/irods/test/harness/003_compile-specific-python.Dockerfile new file mode 100644 index 000000000..63f3ae18c --- /dev/null +++ b/irods/test/harness/003_compile-specific-python.Dockerfile @@ -0,0 +1,16 @@ +FROM ssl-and-pam +RUN apt update +RUN apt install -y wget build-essential +RUN apt install -y libssl-dev zlib1g-dev libffi-dev libncurses-dev wget build-essential +ARG python_version +RUN wget https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tar.xz +RUN tar xf Python-${python_version}.tar.xz +WORKDIR /Python-${python_version} +RUN ./configure --prefix /root/python --with-ensurepip=install +RUN make -j +RUN mkdir /root/python +RUN make install +WORKDIR / +RUN /root/python/bin/python3 -m pip install virtualenv +RUN chmod a+rx /root +ENV PYTHON_VERSION=${python_version} diff --git a/irods/test/harness/README.md b/irods/test/harness/README.md new file mode 100644 index 000000000..de4a33b36 --- /dev/null +++ b/irods/test/harness/README.md @@ -0,0 +1,60 @@ +# Docker Powered Test Harness + +## Description + +A series of docker images which support running isolated test scripts (using BATS, bash, or Python). +Once built, the images allow loading and customizing the Docker container environment for a given +test script. + +The general form for test invocation is: `docker_container_driver.sh ` + +Within the container, a computed internal path to the same script is executed, whether directly or +indirectly by a wrapper script. The wrapper for many of the PRC authentication-via-PAM tests is +irods/test/login_auth_test.sh. + +The test_script_parameters file, located in the irods/test/harness directory, contains customized +settings for each test script run, including: + + - Docker image name to be used. + + - Wrapper to be invoked, if any. Wrappers shall perform common setup tasks up to and including + invoking the test script itself. + + - Which user is running the test. (Unless otherwise specified, this is the passwordless-sudo- + enabled user `testuser`). + +When done with a test, the `docker_container_driver.sh` exit code mirrors the return code from the +run of the test script. The container itself is removed unless the `-L` ("leak") option is given. + +## Sample Runs + +### To build required images + +For our convenience in this doc, set a shell variable `REPOROOT` to `~/python-irodsclient` (or +similar) to specify the path to the top level of the local repository. + +Sample command lines to build Docker images: + +1. ``` + cd $REPO_ROOT/irods/test/harness + ./build_docker.sh + ``` + + Builds docker images in proper sequence. + +2. ``` + cd $REPO_ROOT/irods/test/harness; + IRODS_PACKAGE_VERSION=4.3.4 PYTHON_VERSION=3.11 NO_CACHE=1 ./build-docker.sh [ Dockerfiles... ] + ``` + + Builds (ignoring docker cache) images based on specific iRODS package version and desired + Python Interpreter version, optionally with a restricted list of Docker files in need of rebulding. + +### To run a test script. + +``` +$REPO_ROOT/irods/test/harness/docker_container_driver.sh $REPO_ROOT/irods/test/scripts/run_local_suite +``` + +For both builder and driver script, the environment variable `DOCKER` may be set to `podman` to run +the alternative container engine. Otherwise it default to a value of `docker`. diff --git a/irods/test/harness/build-docker.sh b/irods/test/harness/build-docker.sh new file mode 100755 index 000000000..9c1072692 --- /dev/null +++ b/irods/test/harness/build-docker.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# environment variables for build +# IRODS_PACKAGE_VERSION if defined is like "4.3.4" or "5.0.1". +# (but contains no '~' suffix for irods versions <= 4.2.10) +# PYTHON_VERSION is usually two dot-separated numbers: example "3.13", but could also have zero, one or three version numbers. +# (Do not specify the triple form, X.Y.Z, if that release is not known to exist - not counting alphas and release candidates) + +DIR=$(realpath "$(dirname "$0")") +: ${DOCKER:=docker} + +if [ $# -gt 0 ]; then + IFS=$'\n' read -ra ARGS -d '' < <(realpath --relative-to "$DIR" "$@") + cd "$DIR" +else + cd "$DIR" + ARGS=([0-9]*.Dockerfile) +fi + +: ${PYTHON_VERSION:=3.13} export PYTHON_VERSION + +for dockerfile in "${ARGS[@]}"; do + image_name=${dockerfile#[0-9]*_} + image_name=${image_name%.Dockerfile} + irods_package_version_option="" + python_version_option="" + if [ "$image_name" = "install-irods" ]; then + irods_package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"} + elif [ "$image_name" = "compile-specific-python" ]; then + temp=$(./most_recent_python.sh "$PYTHON_VERSION") + if [ -n "$temp" ]; then + PYTHON_VERSION="$temp" + fi + python_version_option=${PYTHON_VERSION:+"--build-arg=python_version=$PYTHON_VERSION"} + fi + $DOCKER build -f $dockerfile -t $image_name . $irods_package_version_option $python_version_option \ + ${NO_CACHE+"--no-cache"} || + { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; } +done diff --git a/irods/test/harness/create_docker_images.sh b/irods/test/harness/create_docker_images.sh new file mode 100755 index 000000000..2f831a078 --- /dev/null +++ b/irods/test/harness/create_docker_images.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export IRODS_PACKAGE_VERSION=$1 +export PYTHON_VERSION=$2 + +[ -z "$1" -o -z "$2" ] && { + echo >&2 "usage $0 irods-vsn py-vsn"; exit 2; +} +shift 2 + +DIR=$(dirname "$0") + +"$DIR"/build-docker.sh $* diff --git a/irods/test/harness/docker_container_driver.sh b/irods/test/harness/docker_container_driver.sh new file mode 100755 index 000000000..f76cceba4 --- /dev/null +++ b/irods/test/harness/docker_container_driver.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# Runs a test program within a new container. The container is dispatched and/or disposed of, and the exit +# status code of the target test program collected and returned, by this script. + +# The repository containing this harness directory is mapped to a direct subdirectory of / within the container. +# (By present convention that subdirectory is: /prc) The test program to be run is given by its host path, and +# the internal (to the container) path will be computed. + +# The "-L" or leak option may be given as an instruction not to kill or remove the container after the test run. +# A sourced header for this script, 'test_script_parameters', contains configuration for each script that will +# be run under its control. + +KILL_TEST_CONTAINER=1 +RUN_AS_USER="" +ECHO_CONTAINER="" +REMOVE_OPTION="--rm" +EXPLICIT_WORKDIR="" +VERBOSITY=0 +while [[ $1 = -* ]]; do + if [ "$1" = -V ]; then + VERBOSITY=1 + shift + fi + if [ "$1" = -c ]; then + ECHO_CONTAINER=1 + shift + fi + if [ "$1" = -L ]; then + KILL_TEST_CONTAINER=0 + shift + fi + if [ "$1" = -u ]; then + RUN_AS_USER="$2" + shift 2 + fi + if [ "$1" = -r ]; then + REMOVE_OPTION="$2" + shift 2 + fi + if [ "$1" = -w ]; then + EXPLICIT_WORKDIR="$2" + shift 2 + fi +done + +if [ "$1" = "" ]; then + echo >&2 "Usage: $0 [options] /path/to/script" + echo >&2 "With options: [-L] to leak, [-u username] to run as non-root user" + exit 1 +fi + +DIR=$(dirname "$0") +. "$DIR"/test_script_parameters + +testscript=${1} +shift + +testscript_basename=$(basename "$testscript") +arglist=${wrapper_arglist[$testscript_basename]:-$*} # arglist dominated by symbolic link name if any + +if [ -L "$testscript" ]; then + testscript=$(realpath "$testscript") + testscript_basename=$(basename "$testscript") +fi + +original_testscript_abspath=$(realpath "$testscript") + +wrapped=${wrappers["$testscript_basename"]} + +if [ -n "$wrapped" ]; then + # wrapped is assumed to contain a leading path element relative to the referencing script's containing directory + testscript="$(dirname "$testscript")/$wrapped" + testscript_basename=$(basename "$testscript") +fi + +testscript_abspath=$(realpath "$testscript") + +cd "$DIR" + +image=${images[$testscript_basename]} + +if [ -z "$RUN_AS_USER" ]; then + RUN_AS_USER=${user[$testscript_basename]} +fi + +# Tests are run as testuser by default +: ${RUN_AS_USER:='testuser'} + +WORKDIR="" +if [ -n "$EXPLICIT_WORKDIR" ]; then + WORKDIR="$EXPLICIT_WORKDIR" +else + WORKDIR=${workdirs[$RUN_AS_USER]} +fi + +reporoot=$(./print_repo_root_location) +ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to "$reporoot" "$original_testscript_abspath") + +echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" +INNER_MOUNT=/prc + +: ${DOCKER:=docker} + +# Start the container. +echo image="[$image]" +CONTAINER=$($DOCKER run -d -v "$reporoot:$INNER_MOUNT:ro" $REMOVE_OPTION $image) + +# Wait for iRODS and database to start up. +TIME0=$(date +%s) +while :; do + [ $(date +%s) -gt $((TIME0 + 30)) ] && { echo >&2 "Waited too long for DB and iRODS to start"; exit 124; } + sleep 1 + $DOCKER exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null + [ $? -ne 0 ] && { echo -n . >&2; continue; } + break +done + +if [ $VERBOSITY -gt 0 ]; then + echo $'\n'"==> Running script [$testscript_abspath]" + echo "in container [$CONTAINER]" + echo "with these *_VERSION variables in environment: " + $DOCKER exec $CONTAINER bash -c 'env|grep _VERSION' | sed $'s/^/\t/' +fi + +$DOCKER exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \ + ${WORKDIR:+"-w$WORKDIR"} \ + -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \ + $CONTAINER \ + "$INNER_MOUNT/$(realpath --relative-to "$reporoot" "$testscript_abspath")" \ + $arglist +STATUS=$? + +if [ $((0+KILL_TEST_CONTAINER)) -ne 0 ]; then + echo >&2 'Killed:' $($DOCKER stop --timeout=0 $CONTAINER) +fi + +[ -n "$ECHO_CONTAINER" ] && echo $CONTAINER +exit $STATUS diff --git a/irods/test/harness/install.sh b/irods/test/harness/install.sh new file mode 100755 index 000000000..9a43a5dec --- /dev/null +++ b/irods/test/harness/install.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +# A script to manage the main steps in installing an iRODS server as well as all necessary support. (Dependencies, +# catalog database, etc.) + +IRODS_HOME=/var/lib/irods +DEV_HOME="$HOME" +: ${DEV_REPOS:="$DEV_HOME/github"} + +add_package_repo() +{ + echo >&2 "... installing package repo" + sudo apt update + sudo apt install -y lsb-release apt-transport-https gnupg2 + wget -qO - https://packages.irods.org/irods-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import \ + && \ + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods.list + + sudo apt update +} + +# Expand a spec of the leading version tuple eg. 4.3.4 out to the full name of +# the most recent matching version of the package + +# Report the latest version spec (including OS) that matches the env var IRODS_PACKAGE_VERSION (eg. "5.0.2" -> "5.0.2-0~jammy) + +irods_package_vsn() { + apt list -a irods-server 2>/dev/null|awk '{print $2}'|grep '\w'|sort|\ + grep "$(perl -e 'print quotemeta($ARGV[0])' "$IRODS_PACKAGE_VERSION")"|tail -1 +} + +# Report the version number of the installed iRODS server if any. + +irods_vsn() { + local V=$(dpkg -l irods-server 2>/dev/null|grep '^ii\s'|awk '{print $3}') + echo "${V}" +} + +while [[ "$1" = -* ]]; do + ARG="$1" + shift + case $ARG in + --i=* | --irods=* |\ + --irods-version=*) IRODS_PACKAGE_VERSION=${ARG#*=};; + --w=* | --with=* | --with-options=* ) withopts=${ARG#*=} ;; + esac +done + + +run_phase() { + + local PHASE=$1 + local with_opts=" $2 " + + case "$PHASE" in + + 0) + + if [[ $with_opts = *\ initialize\ * ]]; then + apt-get -y update + apt-get install -y apt-transport-https wget lsb-release sudo jq + fi + + if [[ $with_opts = *\ sudo-without-pw\ * ]]; then + if [ $(id -u) = 0 -a "${USER:-root}" = root ] ; then + echo >&2 "root authorization for 'sudo' is automatic - no /etc/sudoers modification needed" + else + if [ -f "/etc/sudoers" ]; then + # add a line with our USER name to /etc/sudoers if not already there + sudo su -c "sed -n '/^\s*[^#]/p' /etc/sudoers | grep '^$USER\s*ALL=(ALL)\s*NOPASSWD:\s*ALL\s*$' >/dev/null" || \ + sudo su -c "echo '$USER ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers" + else + echo >&2 "WARNING - Could not modify sudoers files" + fi + fi # not root + fi # with-opts + + #------ (needed for both package install and build from source) + + if [[ $with_opts = *\ install-essential-packages\ * ]]; then + + if ! dpkg -l tzdata >/dev/null 2>&1 ; then + sudo su - root -c \ + "env DEBIAN_FRONTEND=noninteractive bash -c 'apt-get install -y tzdata'" + fi + sudo apt-get update + sudo apt-get install -y software-properties-common postgresql + sudo apt-get update && \ + sudo apt-get install -y libfuse2 unixodbc rsyslog + fi + + + if [[ $with_opts = *\ add-package-repo\ * ]]; then + add_package_repo -f + fi + + + if [[ $with_opts = *\ create-db\ * ]]; then + sudo su - postgres -c " + { dropdb --if-exists ICAT + dropuser --if-exists irods ; } >/dev/null 2>&1" + sudo su - postgres -c "psql <<\\ +________ + CREATE DATABASE \"ICAT\"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE \"ICAT\" to irods; +________" + echo >&2 "-- status of create-db = $? -- " + fi + ;; + + 4) + IRODS_TO_INSTALL=$(irods_package_vsn) + sudo apt install -y irods-{dev,runtime}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + if [[ $with_opts != *\ basic\ * ]]; then + sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + fi + ;; + + 5) + if [ ! $(irods_vsn) '<' "4.3" ]; then + PYTHON=python3 + else + PYTHON=python2 + fi + sudo $PYTHON /var/lib/irods/scripts/setup_irods.py < /var/lib/irods/packaging/localhost_setup_postgres.input + ;; + + *) echo >&2 "unrecognized phase: '$PHASE'." ; QUIT=1 ;; + esac + return $? +} + +#-------------------------- main + +QUIT=0 +while [ $# -gt 0 ] ; do + ARG=$1 ; shift + NOP="" ; run_phase $ARG " $withopts "; sts=$? + [ $QUIT != 0 ] && break + [ -n "$NOP" ] && continue + echo -n "== $ARG == " + if [ $sts -eq 0 ]; then + echo Y >&2 + else + [ $quit_on_phase_err ] && { echo >&2 "N - quitting"; exit 1; } + echo N >&2 + fi +done diff --git a/irods/test/harness/install_python_rule_engine b/irods/test/harness/install_python_rule_engine new file mode 100755 index 000000000..036273fa1 --- /dev/null +++ b/irods/test/harness/install_python_rule_engine @@ -0,0 +1,23 @@ +#!/bin/bash + +python_rule_plugin_package_spec() { + local PKG=irods-rule-engine-plugin-python + local search_str="+$IRODS_PACKAGE_VERSION" + if [ "$1" = "all" ]; then + search_str="" + fi + local VERSIONS=$(apt list -a $PKG 2>/dev/null|\ + awk '{print $2}'|\ + grep '\w'|\ + grep "$( + perl -e 'print quotemeta($ARGV[0])' "$search_str")" + ) + local LATEST_VERSION=$(sort -V <<<"$VERSIONS" | tail -1) + if [ "$1" = "latest" ]; then + echo "$PKG=$LATEST_VERSION" + else + echo "$VERSIONS" + fi +} + +apt install -y "$(python_rule_plugin_package_spec latest)" diff --git a/irods/test/harness/irods_version_greater_or_equal_to b/irods/test/harness/irods_version_greater_or_equal_to new file mode 100755 index 000000000..a5375b144 --- /dev/null +++ b/irods/test/harness/irods_version_greater_or_equal_to @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import ast +import getopt +import glob +import json +import os +import sys + +def version_to_tuple(dot_str): + return tuple(int(_) for _ in dot_str.split('.')) + +# Fail unless iRODS version is greater than or equal to a version string given on the command line. +# With -e, we determine the installed iRODS version from the specified environment variable. +# Else use ~irods/version.json + +if __name__ == '__main__': + opts,arg = getopt.getopt (sys.argv[1:],'e:',['use_env_var=']) + + env_var = '' + + for opt,val in opts: + if opt in {'-e','--use_env_var'}: + env_var = val + + pattern = os.path.join(os.path.expanduser('~irods'), "*.json") + + # get full path of version.json + + if env_var: + version_to_test = os.environ[env_var] + else: + version_files = list( + filter( + (lambda name: name.lower().endswith('version.json')), + glob.glob(pattern) + ) + ) + # Load JSON struct containing iRODS version. + j = json.load(open(version_files[0])) + version_to_test = j["irods_version"] + + if version_to_tuple(version_to_test) < version_to_tuple(arg[0]): + exit(1) diff --git a/irods/test/harness/manage_irods5_procs b/irods/test/harness/manage_irods5_procs new file mode 100755 index 000000000..3327cb9be --- /dev/null +++ b/irods/test/harness/manage_irods5_procs @@ -0,0 +1,57 @@ +if [ `id -un` = irods ]; then + LAUNCH='bash -c' +else + LAUNCH='sudo su - irods -c' +fi + +STDOUT="" +PID="" + +start() { + if [ -z "$STDOUT" ] ; then + $LAUNCH 'irodsServer -d -p /tmp/irods.pid' + else + $LAUNCH 'irodsServer --stdout -p /tmp/irods.pid >/tmp/irods.log &' + fi +} + +rm_pid_file() { + if [ -z "$PID" ]; then + PID=$($LAUNCH 'cat /tmp/irods.pid') + fi + $LAUNCH 'rm -f /tmp/irods.pid >/dev/null 2>&1' +} + +stop() { + $LAUNCH 'kill -QUIT $(cat /tmp/irods.pid)' + rm_pid_file +} + +wait() { + $LAUNCH " + [ -z '$PID' ] && { echo >&2 'nothing to wait for.' ; exit 2; } + while ps -eo pid |grep $PID >/dev/null 2>&1; do sleep 1; done;" +} + +# ----------------------------------- +while [ -n "$1" ]; do + if [ "$1" = "stdout" ]; then + STDOUT=1 + elif [ "$1" = "start" ]; then + start + elif [ "$1" = "rescan-config" ]; then + $LAUNCH 'pkill -HUP irodsServer' + elif [ "$1" = "status" ]; then + pgrep -afl "irods(Delay|Agent|Server)" + elif [ "$1" = "stop" ]; then + stop + elif [ "$1" = "restart" ]; then + stop && start + elif [ "$1" = "wait" ]; then + wait + else + echo >&2 "usage: $0 [start|status|stop]" + exit 2 + fi + shift +done diff --git a/irods/test/harness/most_recent_python.sh b/irods/test/harness/most_recent_python.sh new file mode 100755 index 000000000..278f3ce7c --- /dev/null +++ b/irods/test/harness/most_recent_python.sh @@ -0,0 +1,25 @@ +#!/bin/bash +usage() { + echo >&2 "Usage: + $0 major.minor" + echo >&2 "Output: + prints full latest python version inclusive of the patch level." + exit 2 +} +MAJOR_MINOR=$1 +if [ -z "${MAJOR_MINOR}" ]; then # allow blank specification: most recent overall + MAJOR_MINOR='[0-9]\+\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ ^[0-9]+$ ]]; then # allow single integer, eg. 3 for most recent 3.y.z + MAJOR_MINOR+='\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ [0-9]+\.[0-9]+ ]]; then # allow x.y form, will yield output of most recent x.y.z + MAJOR_MINOR=$(sed 's/\./\\./'<<<"${MAJOR_MINOR}") # insert backslash in front of "." +elif ! [[ $MAJOR_MINOR =~ [0-9]+\\?.[0-9]+ ]]; then + usage +fi + +url='https://www.python.org/ftp/python/' + +# Fetch the directory listing, extract version numbers, sort them to find the largest numerically. +curl --silent "$url"|\ +sed -n 's!.*href="\('"${MAJOR_MINOR}"'\.[0-9]\+\)/".*!\1!p'|sort -rV|\ +head -n 1 diff --git a/irods/test/harness/print_repo_root_location b/irods/test/harness/print_repo_root_location new file mode 100755 index 000000000..adcf3fc73 --- /dev/null +++ b/irods/test/harness/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../../.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/irods/test/harness/setup_python_rule_engine b/irods/test/harness/setup_python_rule_engine new file mode 100755 index 000000000..6795f04e5 --- /dev/null +++ b/irods/test/harness/setup_python_rule_engine @@ -0,0 +1,92 @@ +#!/bin/bash + +# This script should be run as the service account user. + +wait="" +if [ $1 = --wait ]; then + wait=1 + shift +fi + +DIR=$(dirname "$0") + +server_ctl() { + # This script takes one argument. + # Valid arguments: start, stop, or restart. The appropriate action is then taken for the resident iRODS server. + if "$DIR"/irods_version_greater_or_equal_to --use_env_var=IRODS_PACKAGE_VERSION 5.0; then + # Make our ps-based script wait for process shutdown like 'irodsctl stop' does. + W="" + if [ "$1" = stop ]; then + W=wait + fi + /manage_irods5_procs $1 $W + else + ~/irodsctl $1 + fi +} + +jq_process_in_place() { + local filename=$1 + shift + local basenm=$(basename "$filename") + local tempname=/tmp/.$$.$basenm + + jq "$@" <"$filename" >"$tempname" && \ + cp "$tempname" "$filename" + STATUS=$? + rm -f "$tempname" + [ $STATUS = 0 ] || echo "**** jq process error" >&2 +} + +# -- Main part of script -- + +server_ctl stop + +jq_process_in_place /etc/irods/server_config.json \ + '.plugin_configuration.rule_engines[1:1]=[ { "instance_name": "irods_rule_engine_plugin-python-instance", + "plugin_name": "irods_rule_engine_plugin-python", + "plugin_specific_configuration": {} + } + ]' +echo ' +defined_in_both { + writeLine("stdout", "native rule") +} + +generic_failing_rule { + fail +} + +failing_with_message { + failmsg(-2, "error with code of minus 2") +} + +' >> /etc/irods/core.re + +echo ' +def defined_in_both(rule_args,callback,rei): + callback.writeLine("stdout", "python rule") + +def generic_failing_rule(*_): + raise RuntimeError + +def failing_with_message_py(rule_args,callback,rei): + callback.failing_with_message() + +' > /etc/irods/core.py + +server_ctl start + +# Wait until 'irule -a' shows Python Rule Engine Plugin among the choices +if [ -n "$wait" ]; then + times=0 + OUTFILE=/tmp/irule_output.stderr + while :; do + irule -a 2>/dev/null| grep irods_rule_engine_plugin-python-instance >/dev/null + [ ${PIPESTATUS[1]} -eq 0 ] && break + sleep 1 + if [ $((++times)) -ge 10 ]; then + echo >&2 "Failed to configure Python rule engine."; exit 2; + fi + done +fi diff --git a/irods/test/harness/start_postgresql_and_irods.sh b/irods/test/harness/start_postgresql_and_irods.sh new file mode 100755 index 000000000..93f289b38 --- /dev/null +++ b/irods/test/harness/start_postgresql_and_irods.sh @@ -0,0 +1,34 @@ +#!/bin/bash +service postgresql start +x=${DB_WAIT_SEC:-20} +while [ $x -ge 0 ] && { ! $SUDO su - postgres -c "psql -c '\l' >/dev/null 2>&1" || x=""; } +do + [ -z "$x" ] && break + echo >&2 "$((x--)) secs til database timeout"; sleep 1 +done +[ -z "$x" ] || { echo >&2 "Error -- database didn't start" ; exit 1; } +VERSION_file=$(ls /var/lib/irods/{VERSION,version}.json.dist 2>/dev/null) +if ! id -u irods >/dev/null 2>&1 ; then + /install.sh --w=create-db 0 + /install.sh 5 +fi +IRODS_VSN=$(jq -r '.irods_version' $VERSION_file) +IRODS_VSN_MAJOR=${IRODS_VSN//.*/} +if [ "$IRODS_VSN_MAJOR" -lt 5 ]; then + su - irods -c '~/irodsctl restart' +else + /manage_irods5_procs start +fi +IRODS_WAIT_SEC=20 +x=$IRODS_WAIT_SEC +SLEEPTIME="" +while [ $((x--)) -gt 0 ]; do + sleep $((SLEEPTIME+0)) + pgrep irodsServer + STATUS=$? + [ $STATUS -eq 0 ] && break + SLEEPTIME=1 +done +echo "($STATUS)" >/tmp/irods_status +[ $STATUS -eq 0 ] || exit 125 +tail -f /dev/null diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters new file mode 100644 index 000000000..fe2beca54 --- /dev/null +++ b/irods/test/harness/test_script_parameters @@ -0,0 +1,44 @@ +# keys for Arglist refer to argument given, which could be a symlink. + +declare -A wrapper_arglist=( + [login_auth_test_must_run_manually.py]="-v TestLogins" + [login_auth_test_1.py]="-v TestAnonymousUser TestMiscellaneous" + [login_auth_test_2.py]="-v TestWithSSL" +) + +# keys for Wrapper refer to argument after resolution of any symlinks + +declare -A wrappers=( + [login_auth_test_must_run_manually.py]=./login_auth_test.sh + [PRC_issue_362.bats]=./login_auth_test.sh + [test001_pam_password_expiration.bats]=../login_auth_test.sh + [test002_write_native_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test003_write_pam_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test004_prc_pam_password_internal_secrets_file_generation.bats]=../login_auth_test.sh + [test005_test_special_characters_in_pam_passwords.bats]=../login_auth_test.sh + [test006_connection_timeout_on_ssl_socket.bats]=../login_auth_test.sh + [test007_pam_features_in_new_auth_framework.bats]=../login_auth_test.sh + [test008_prc_write_irodsA_utility_in_native_mode.bats]=../login_auth_test.sh + [test009_test_special_characters_in_pam_passwords_auth_framework.bats]=../login_auth_test.sh + [test010_issue_362_rogue_chars_in_pam_password.bats]=../login_auth_test.sh +) + +# keys for Image and User refer to the basename after resolution to a wrapper if one is used + +declare -A images=( + [login_auth_test.sh]=compile-specific-python + [login_auth_test_must_run_manually.py]=ssl-and-pam + [run_suite_locally.sh]=compile-specific-python +) + +declare -A user=( + [run_suite_locally.sh]=root +) + +# keys for WorkDir refer to user + +declare -A workdirs=( + [testuser]=/home/testuser + [irods]=/var/lib/irods + [root]=/ +) diff --git a/irods/test/login_auth_test.sh b/irods/test/login_auth_test.sh new file mode 100755 index 000000000..497cdc81d --- /dev/null +++ b/irods/test/login_auth_test.sh @@ -0,0 +1,77 @@ +#!/bin/bash +. "$(dirname "$0")/scripts/test_support_functions" +. "$(dirname "$0")/scripts/update_json_for_test" + +IRODS_SERVER_CONFIG=/etc/irods/server_config.json +IRODS_SERVICE_ACCOUNT_ENV_FILE=~irods/.irods/irods_environment.json +LOCAL_ACCOUNT_ENV_FILE=~/.irods/irods_environment.json + +cannot_iinit='' +tries=8 +while true; do + iinit_as_rods >/dev/null 2>&1 && break + [ $((--tries)) -le 0 ] && { cannot_iinit=1; break; } + sleep 5 +done +[ -n "$cannot_iinit" ] && { echo >&2 "Could not iinit as rods."; exit 2; } + +setup_preconnect_preference DONT_CARE + +add_irods_to_system_pam_configuration + + +# set up /etc/irods/ssl directory and files +set_up_ssl sudo + +sudo useradd -ms/bin/bash alissa +sudo chpasswd <<<"alissa:test123" + +update_json_file $IRODS_SERVICE_ACCOUNT_ENV_FILE \ + "$(newcontent $IRODS_SERVICE_ACCOUNT_ENV_FILE ssl_keys)" + +# This is mostly so we can call python3 as just "python" +activate_virtual_env_with_prc_installed >/dev/null 2>&1 || { echo >&2 "couldn't set up virtual environment"; exit 1; } + +server_hup= +if irods_server_version ge 5.0.0; then + server_hup="y" + update_json_file $IRODS_SERVER_CONFIG \ + "$(newcontent $IRODS_SERVER_CONFIG tls_server_items tls_client_items)" + + sudo su - irods -c "/manage_irods5_procs rescan-config" +fi + +# Configure clients with admin user + TLS + +update_json_file $LOCAL_ACCOUNT_ENV_FILE \ + "$(newcontent $LOCAL_ACCOUNT_ENV_FILE ssl_keys encrypt_keys)" + +if [ "$server_hup" = y ]; then + # wait for server to be ready after configuration reload + while true; do + sleep 2 + if ils >/dev/null 2>&1; then + break + else + # Allow ~16 secs of total wait time. + [ $((++server_check)) -gt 8 ] && { + echo >&2 "Timed out on server reload"; exit 3; } + fi + done +fi + +if [ -n "$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" ]; then + original_script="/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" + + # Run tests. + if [ -x "$original_script" ]; then + command "$original_script" $* + elif [[ $original_script =~ \.py$ ]]; then + python "$original_script" $* + elif [[ $original_script =~ \.bats$ ]]; then + bats "$original_script" + else + echo >&2 "I don't know how to run this: original_script=[$original_script]" + fi + +fi diff --git a/irods/test/login_auth_test_1.py b/irods/test/login_auth_test_1.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_1.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_2.py b/irods/test/login_auth_test_2.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_2.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_must_run_manually.py b/irods/test/login_auth_test_must_run_manually.py index 4303e47c1..745eaf53e 100644 --- a/irods/test/login_auth_test_must_run_manually.py +++ b/irods/test/login_auth_test_must_run_manually.py @@ -534,7 +534,7 @@ def test_nonanonymous_login_without_auth_file_fails__290(self): s.users.get("bob") os.unlink(bob_auth) # -- Check that we raise an appropriate exception pointing to the missing auth file path -- - with self.assertRaisesRegexp(NonAnonymousLoginWithoutPassword, bob_auth): + with self.assertRaisesRegex(NonAnonymousLoginWithoutPassword, bob_auth): with helpers.make_session(**login_options) as s: s.users.get("bob") finally: diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index aa33dae26..1a0d01bf4 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -683,16 +683,16 @@ def test_AVUs_populated_improperly_with_empties_or_nonstrings_fail_identically__ def test_nonstring_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", 0) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.add(*args) def test_empty_string_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", "") - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.add(*args) @unittest.skipUnless( @@ -724,10 +724,11 @@ def test_that_all_column_mappings_are_uniquely_and_properly_defined__issue_643( prepend_col_prefix_if_needed = lambda s: ( "COL_" + s if not s.startswith("COL_") else s ) + current_server_version = self.sess.server_version prc_column_defs = sorted( [ (prepend_col_prefix_if_needed(i[1].icat_key), i[1].icat_id) - for i in ModelBase.column_items + for i in ModelBase.column_items if current_server_version >= i[1].min_version ] ) diff --git a/irods/test/pam.bats/funcs b/irods/test/pam.bats/funcs deleted file mode 100644 index 30539a03a..000000000 --- a/irods/test/pam.bats/funcs +++ /dev/null @@ -1,108 +0,0 @@ -dot_to_space() { - sed 's/\./ /g'<<<"$1" -} - -CLEANUP=$':\n' - -GT() { (return 1); echo $?; } -LT() { (return -1); echo $?; } -EQ() { (return 0); echo $?; } - -compare_int_tuple() { - local x=($1) y=($2) - local lx=${#x[@]} ly=${#y[@]} - local i maxlen=$((lx > ly ? lx : ly)) - for ((i=0;i ~/.irods/irods_environment.json - iinit <<<"$1" 2>/dev/tty -} - -_end_pam_environment_and_password() { - rm -fr ~/.irods - mv ~/.irods.$$ ~/.irods -} - -setup_pam_login_for_alice() { - sudo useradd alice --create-home - local PASSWD=${1:-test123} - sudo chpasswd <<<"alice:$PASSWD" - iadmin mkuser alice rodsuser - _begin_pam_environment_and_password "$PASSWD" -} - -finalize_pam_login_for_alice() { - _end_pam_environment_and_password - iadmin rmuser alice - sudo userdel alice --remove -} - -test_specific_cleanup() { - eval "$CLEANUP" -} diff --git a/irods/test/pam.bats/test001_pam_password_expiration.bats b/irods/test/pam.bats/test001_pam_password_expiration.bats deleted file mode 100644 index 3e29100ef..000000000 --- a/irods/test/pam.bats/test001_pam_password_expiration.bats +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bats - -. "$BATS_TEST_DIRNAME"/test_support_functions -PYTHON=python3 - -# Setup/prerequisites are same as for login_auth_test. -# Run as ubuntu user with sudo; python_irodsclient must be installed (in either ~/.local or a virtualenv) -# - -PASSWD=test123 - -setup() -{ - setup_pam_login_for_alice $PASSWD -} - -teardown() -{ - finalize_pam_login_for_alice - test_specific_cleanup -} - -@test f001 { - - # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS - # without an exception being raised. - - local SCRIPT=" -import irods.test.helpers as h -ses = h.make_session() -ses.collections.get(h.home_collection(ses)) -print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) -" - - # Test that the first run of the code in $SCRIPT is successful, i.e. normal authenticated operations are possible. - - local OUTPUT=$($PYTHON -c "$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - SET_CLEANUP=yes \ - with_change_auth_params_for_test password_min_time 4 \ - password_max_time 5 - - # Test that running the $SCRIPT raises an exception if the PAM password has expired. - - iinit <<<"$PASSWD" - HOME_COLLECTION=$(ipwd) - sleep 9 - OUTPUT=$($PYTHON -c "$SCRIPT" 2>&1 >/dev/null || true) - grep 'RuntimeError: Time To Live' <<<"$OUTPUT" - - # Test that the $SCRIPT, when run with proper settings, can successfully reset the password. - - with_change_auth_params_for_test password_max_time 3600 - - OUTPUT=$($PYTHON -c "import irods.client_configuration as cfg -cfg.legacy_auth.pam.password_for_auto_renew = '$PASSWD' -cfg.legacy_auth.pam.time_to_live_in_hours = 1 -cfg.legacy_auth.pam.store_password_to_environment = True -$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - # Test that iCommands can authenticate with the newly written .irodsA file - - iquest "%s" "select COLL_NAME where COLL_NAME like '%/home/alice%'"| grep "^$HOME_COLLECTION\$" -} diff --git a/irods/test/pam_interactive_test.py b/irods/test/pam_interactive_test_must_run_manually.py similarity index 100% rename from irods/test/pam_interactive_test.py rename to irods/test/pam_interactive_test_must_run_manually.py diff --git a/irods/test/rule_test.py b/irods/test/rule_test.py index be95302b7..39b718a26 100644 --- a/irods/test/rule_test.py +++ b/irods/test/rule_test.py @@ -20,7 +20,7 @@ RE_Plugins_installed_run_condition_args = ( os.environ.get("PYTHON_RULE_ENGINE_INSTALLED", "*").lower()[:1] == "y", - "Test depends on server having Python-REP installed beyond the default options", + "Test depends on server having Python-REP installed (set PYTHON_RULE_ENGINE_INSTALLED=yes in environment)." ) @@ -420,7 +420,7 @@ def test_rulefile_in_file_like_object_1__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r".*\[Hello world!\]") + self.assertRegex(lines[0], r".*\[Hello world!\]") def test_rulefile_in_file_like_object_2__336(self): @@ -442,8 +442,8 @@ def test_rulefile_in_file_like_object_2__336(self): r = Rule(self.sess, rule_file=io.BytesIO(rule_file_contents.encode("utf-8"))) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[STRING\]\[\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[\]") + self.assertRegex(lines[0], r"\[STRING\]\[\]") + self.assertRegex(lines[1], r"\[STRING\]\[\]") r = Rule( self.sess, @@ -452,8 +452,8 @@ def test_rulefile_in_file_like_object_2__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[INTEGER\]\[5\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[A String\]") + self.assertRegex(lines[0], r"\[INTEGER\]\[5\]") + self.assertRegex(lines[1], r"\[STRING\]\[A String\]") if __name__ == "__main__": diff --git a/irods/test/runner.py b/irods/test/runner.py index f9f9fa610..8782d6391 100644 --- a/irods/test/runner.py +++ b/irods/test/runner.py @@ -7,6 +7,7 @@ """ +import argparse import os import sys from unittest import TestLoader, TestSuite @@ -22,20 +23,74 @@ h.setFormatter(f) logger.addHandler(h) +parser = argparse.ArgumentParser() -# Load all tests in the current directory and run them +def abs_path(initial_dir, levels_up = 0): + directory = initial_dir + while levels_up > 0: + levels_up -= 1 + directory = os.path.join(directory,'..') + return os.path.abspath(directory) + +# Load all tests in the current directory and run them. if __name__ == "__main__": - # must set the path for the imported tests - sys.path.insert(0, os.path.abspath("../..")) + + # Get path to script directory for test import and/or discovery. + script_dir = os.path.abspath(os.path.dirname(sys.argv[0])) + + # Must set the path for the imported tests. + sys.path.insert(0, abs_path(script_dir, levels_up = 2)) + + parser.add_argument('--tests', '-t', + metavar='TESTS', + dest='tests', + nargs='+', + help='List of tests to run.') + + parser.add_argument('--environment_variable', '-e', + metavar='ENVIRONMENT_VARIABLE', + dest='env_var', + type=str, + help='Name of environment variable name to scan for in reason strings when filtering skipped test names to be output.') + + parser.add_argument('--output_tests_skipped', '-s', + metavar='SKIPPED_TESTS_OUTPUT_FILENAME', + dest='skipped_tests_output_filename', + type=str, + help='Name of a file into which to write names of skipped tests.') + + parser.add_argument('--tests_file', '-f', + metavar='TESTS_FILE', + dest='tests_file', + help='Name of a file containing a list of tests to run.') + + args = parser.parse_args() + + if args.tests_file: + if args.tests: + print ('Cannot specify both --tests and --tests_file', file = sys.stderr) + exit(2) + args.tests = filter(None,open(args.tests_file).read().split("\n")) loader = TestLoader() - suite = TestSuite( - loader.discover(start_dir=".", pattern="*_test.py", top_level_dir=".") - ) + + if args.tests: + suite = TestSuite(loader.loadTestsFromNames(args.tests)) + else: + suite = TestSuite(loader.discover(start_dir = script_dir, pattern = '*_test.py', top_level_dir = script_dir)) result = xmlrunner.XMLTestRunner( verbosity=2, output="/tmp/python-irodsclient/test-reports" ).run(suite) + + if args.skipped_tests_output_filename: + with open(args.skipped_tests_output_filename,'w') as skip_file: + do_output = (lambda reason: (args.env_var in reason) if args.env_var + else True) + for testinfo, reason in result.skipped: + if do_output(reason): + print(testinfo.test_id, file=skip_file) + if result.wasSuccessful(): sys.exit(0) diff --git a/irods/test/scripts/iinit.py b/irods/test/scripts/iinit.py new file mode 120000 index 000000000..cb4b84b21 --- /dev/null +++ b/irods/test/scripts/iinit.py @@ -0,0 +1 @@ +../../../docker-testing/iinit.py \ No newline at end of file diff --git a/irods/test/scripts/run_suite_locally.sh b/irods/test/scripts/run_suite_locally.sh new file mode 100755 index 000000000..79f02bdf8 --- /dev/null +++ b/irods/test/scripts/run_suite_locally.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR=$(dirname "$0") +. "$SCRIPT_DIR"/test_support_functions + +run_tests() { + setup_pyN + su - testuser -c " + set -e + source /pyN/bin/activate + pip install -e /prc.rw[tests] + cd /prc.rw/irods/test + python /prc.rw/docker-testing/iinit.py \ + host localhost \ + port 1247 \ + user rods \ + zone tempZone \ + password rods + echo ; echo 'PRC under test: === iRODS [$IRODS_PACKAGE_VERSION] ; Python [$PYTHON_VERSION]' + python runner.py --output_tests_skipped /tmp/skipped.txt -e PYTHON_RULE_ENGINE_INSTALLED --tests irods.test.rule_test + " + + # Install PREP (Python Rule Engine Plugin). + ( + set -e + cd "$SCRIPT_DIR/../harness" + apt update + ./install_python_rule_engine + su irods -c './setup_python_rule_engine --wait' + ) + + # Run PREP-dependent tests that were previously skipped. + su - testuser -c " + set -e + source /pyN/bin/activate + cd /prc.rw/irods/test + env PYTHON_RULE_ENGINE_INSTALLED=yes python runner.py --tests_file /tmp/skipped.txt + " +} + +run_tests diff --git a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats index acf9c9594..8d84ddf82 100755 --- a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats +++ b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats @@ -43,9 +43,13 @@ except irods.client_init.irodsA_already_exists: [ -n "$CONTENTS1" -a "$CONTENTS1" = "$CONTENTS2" ] # Now delete the already existing irodsA and repeat without negating overwrite. + TIMESTAMP_0=$(stat -c%Y $auth_file) + sleep 2 $PYTHON -c "import irods.client_init; irods.client_init.write_pam_irodsA_file('$ALICES_NEW_PAM_PASSWD')" - CONTENTS3=$(cat $auth_file) - [ "$CONTENTS2" != "$CONTENTS3" ] + TIMESTAMP=$(stat -c%Y $auth_file) + + # Test only the timestamp of the new auth_file, not the content, since that is implicitly asserted by the next step. + [ $(($TIMESTAMP-TIMESTAMP_0)) -ge 1 ] # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS # without an exception being raised. diff --git a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats index 23aecd8ce..de9304461 100755 --- a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats +++ b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats @@ -48,6 +48,9 @@ print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) # Write another .irodsA prc_write_irodsA.py native <<<"rods" + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_client_server_policy"]="CS_NEG_REFUSE"' <$CLIENT_JSON >/tmp/client_json_test008.$$ + mv /tmp/client_json_test008.$$ $CLIENT_JSON # Verify new .irodsA for both iCommands and PRC use. ils >/tmp/stdout OUTPUT=$($PYTHON -c "$SCRIPT") diff --git a/irods/test/PRC_issue_362.bats b/irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats old mode 100644 new mode 100755 similarity index 64% rename from irods/test/PRC_issue_362.bats rename to irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats index c3a1ef80f..308b0af8e --- a/irods/test/PRC_issue_362.bats +++ b/irods/test/scripts/test010_issue_362_rogue_chars_in_pam_password.bats @@ -1,15 +1,26 @@ +#!/usr/bin/env bats + # The tests in this BATS module must be run as a (passwordless) sudo-enabled user. # It is also required that the python irodsclient be installed under irods' ~/.local environment. -. $BATS_TEST_DIRNAME/scripts/test_support_functions +. $BATS_TEST_DIRNAME/test_support_functions setup() { - - iinit_as_rods - - setup_pam_login_for_user "test123" alice - - cat >~/test_get_home_coll.py <<-EOF + [ -f /tmp/once ] || { + rm -fr ~/.irods + $BATS_TEST_DIRNAME/iinit.py host localhost \ + port 1247 \ + zone tempZone \ + user rods \ + password rods \ + ## Because iRODS 5+ negotiates for SSL automatically: + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_client_server_policy"]="CS_NEG_REFUSE"' >$CLIENT_JSON.$$ <$CLIENT_JSON + mv $CLIENT_JSON.$$ $CLIENT_JSON + + setup_pam_login_for_user "test123" alice + + cat >~/test_get_home_coll.py <<-EOF import irods.test.helpers as h ses = h.make_session() home_coll = h.home_collection(ses) @@ -17,11 +28,8 @@ setup() { and ses.pool.account._original_authentication_scheme.lower() in ('pam','pam_password') else 1) EOF -} - -teardown() { - iinit_as_rods - finalize_pam_login_for_user alice + } + touch /tmp/once } prc_test() diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions new file mode 100644 index 000000000..f4df0a981 --- /dev/null +++ b/irods/test/scripts/test_support_functions @@ -0,0 +1,245 @@ +# This is effectively a group of utility functions to be sourced from tests +# written in BATS or in straight Bash. Many or most have to do either with: +# +# 1. implementing common tasks, most often setup or configuration wrt iRODS, or +# 2. more primitive functions e.g. string manipulations or comparisons, etc. + +SCRIPTDIR=${BASH_SOURCE[0]} +up_from_script_dir() { + local x incr="" + for ((x=0;x<${1:-0};x++)); do incr+="/.."; done + realpath "$(dirname "$SCRIPTDIR")""$incr" +} + +# Sample usages: +# By user irods: set_up_ssl "" "-q" +# By sudo enabled user: set_up_ssl "sudo" "-q" +set_up_ssl() { + local SUDO=${1:-""} + local OPTS=${2:-""} + $SUDO su - irods -c "python3 $(up_from_script_dir 1)/setupssl.py $OPTS" +} + +# Clears out environment and resets to rodsadmin 'rods'. +# Meant mostly to allow initial steps by a rodsadminfor setting up tests. + +iinit_as_rods() { + rm -fr ~/.irods + iinit <<<$(hostname)$'\n1247\nrods\ntempZone\nrods' +} + +dot_to_space() { + sed 's/\./ /g'<<<"$1" +} + +CLEANUP=$':\n' + +GT() { (return 1); echo $?; } +LT() { (return -1); echo $?; } +EQ() { (return 0); echo $?; } + +compare_int_tuple() { + local x=($1) y=($2) + local lx=${#x[@]} ly=${#y[@]} + local i maxlen=$((lx > ly ? lx : ly)) + for ((i=0;i ~/.irods/irods_environment.json + + # TODO: check: it seems /dev/tty won't work if docker exec is not invoked with -t + if [ -n "$1" -a -z "$SKIP_IINIT_FOR_PASSWORD" ]; then + iinit <<<"$1" 2>/tmp/iinit_as_alice.log + fi +} + +_end_pam_environment_and_password() { + rm -fr ~/.irods + mv ~/.irods.$$ ~/.irods +} + +setup_pam_login_for_user() { + local user=${2:-alice} + sudo useradd $user --create-home + local PASSWD=${1:-test123} + sudo chpasswd <<<"$user:$PASSWD" + iadmin mkuser $user rodsuser + _begin_pam_environment_and_password "$PASSWD" $user +} + +setup_pam_login_for_alice() { + setup_pam_login_for_user "$1" alice +} + +finalize_pam_login_for_user() { + local USER=${1} + _end_pam_environment_and_password + iadmin rmuser "$USER" + sudo userdel "$USER" --remove +} + +finalize_pam_login_for_alice() { + finalize_pam_login_for_user alice +} + +test_specific_cleanup() { + eval "$CLEANUP" +} + +# PostgreSQL only +age_out_pam_password() { + # sets create_ts and modify_ts (timestamps) to older values, decreasing them by an amount of (offset + 1) where offset + # is the number of seconds for expiry_ts stored in the ICAT for the given user and password. In this way, we can + # artificially age out an existing pam password. + # Parameters: + # $1 - The username + # $2 - (optional) override the amount used for offsetting the create & modify timestamps. + local id=$(iquest %s "select USER_ID where USER_NAME = '$1'") + local offset=$(sudo su - postgres -c "psql -t ICAT -c 'select pass_expiry_ts from r_user_password where user_id = $id'") + local mtime=$(sudo su - postgres -c "psql -t ICAT -c 'select modify_ts from r_user_password where user_id = $id'") + mtime=$(sed 's/^\s*0//' <<<"$mtime") + [ -n "$2" ] && offset="$2" + ((offset+=1)) + local new_time=$((mtime - offset)) + sudo su - postgres -c "psql ICAT -c 'update r_user_password set create_ts=$new_time, modify_ts=$new_time where user_id=$id'" +} + +call_irodsctl() { + local arg=${1:-restart} + sudo su - irods -c "./irodsctl $arg" +} + +add_irods_to_system_pam_configuration() { + local tempfile=/tmp/irods-pam-config.$$ + cat <<-EOF >$tempfile + auth required pam_env.so + auth sufficient pam_unix.so + auth requisite pam_succeed_if.so uid >= 500 quiet + auth required pam_deny.so + EOF + sudo chown root.root $tempfile + sudo mv $tempfile /etc/pam.d/irods +} + +setup_preconnect_preference() { + sudo su irods -c "sed -i.orig 's/\(^\s*acPreConnect.*CS_NEG\)\([A-Z_]*\)/\1_$1/' /etc/irods/core.re" +} + +setup_pyN() { + if [ ! -d /pyN ]; then + mkdir /pyN ; chown testuser /pyN + su - testuser -c "/root/python/bin/python3 -m virtualenv /pyN" + + # TODO, REMOVE: Verbose to check proper version is being installed + echo "/pyN venv is python version => [$(. /pyN/bin/activate ; python -V)]" + + cp -r /prc{,.rw} + chown -R testuser /prc.rw + fi +} + +# requires image to descend from bats-python3 +activate_virtual_env_with_prc_installed() +{ + local py_venv=${1:-pyN} + [ "$py_venv" = pyN ] && sudo bash -c "$(declare -f setup_pyN); setup_pyN" + # install python client using copy of /prc so that bdist doesn't build in the readonly mount + sudo su - -c "source /${py_venv}/bin/activate && cp -rp /prc /prc-copy && \ + pip install '/prc-copy[tests]' && sudo rm -fr /prc-copy" + source /${py_venv}/bin/activate + echo "---> Python virtual environment activated. Interpreter Version is: $(python -V)" +} + +mtime_and_content() +{ + stat -c%y "$1" + cat "$1" +} + +irods_server_version() { + python -c "import irods.helpers as h +import operator,sys +if len(sys.argv) == 1: + (comparison,relto)=('','') +elif len(sys.argv) == 3: + (comparison,relto)=sys.argv[1:3] +fm_tuple = lambda tup: '.'.join(str(_) for _ in tup) +to_tuple = lambda vstr: tuple(int(_) for _ in vstr.split('.')) +svt = h.make_session().server_version_without_auth() +if relto: + exit(0 if vars(operator)[comparison](svt,to_tuple(relto)) else 1) +print(fm_tuple(svt)) +" $1 $2 +} diff --git a/irods/test/scripts/update_json_for_test b/irods/test/scripts/update_json_for_test new file mode 100644 index 000000000..3372fd48e --- /dev/null +++ b/irods/test/scripts/update_json_for_test @@ -0,0 +1,69 @@ +#!/bin/bash + +declare -A tls_server_items=( + [tls_server]='{"certificate_chain_file":"/etc/irods/ssl/irods.crt", + "certificate_key_file":"/etc/irods/ssl/irods.key", + "dh_params_file":"/etc/irods/ssl/dhparams.pem"}' +) + +declare -A tls_client_items=( + [tls_client]='{"ca_certificate_file":"/etc/irods/ssl/irods.crt", + "ca_certificate_path":"/etc/ssl/certs", + "verify_server":"cert"}' +) + +declare -A ssl_keys=( + [irods_client_server_negotiation]='"request_server_negotiation"' + [irods_client_server_policy]='"CS_NEG_REQUIRE"' + [irods_ssl_ca_certificate_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_chain_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_key_file]='"/etc/irods/ssl/irods.key"' + [irods_ssl_dh_params_file]='"/etc/irods/ssl/dhparams.pem"' + [irods_ssl_verify_server]='"cert"' +) + +declare -A pam_keys=( + [irods_authentication_scheme]="\"$(pam_auth_string)\"" +) + +declare -A encrypt_keys=( + [irods_encryption_key_size]=16 + [irods_encryption_salt_size]=8 + [irods_encryption_num_hash_rounds]=16 + [irods_encryption_algorithm]='"AES-256-CBC"' +) + +declare -A RESTORE_FILES=() + +update_json_file() { + local file=$1 content=$2 + local bn=$(basename "$file") + local orig=/tmp/$bn.orig.$$ + local newfile=/tmp/$bn.new.$$ + echo "$content" >"$newfile" + sudo chmod --reference "$file" "$newfile" + sudo chown --reference "$file" "$newfile" + { sudo mv "$file" "$orig" && sudo mv "$newfile" "$file"; } || return 1 + RESTORE_FILES["$file"]="$orig" +} + +restore_json_files() { + local kk + for kk in ${!RESTORE_FILES[@]};do + sudo mv -f "${RESTORE_FILES["$kk"]}" "$kk" + done +} + +newcontent () { + local file=$1 + shift + local j=$(sudo cat "$file") + while [ $# -gt 0 ]; do + eval ' + for kk in ${!'$1'[@]}; do + j=$(jq ".$kk=${'$1'[$kk]}" <<<"$j") + done' + shift + done + echo "$j" +} diff --git a/irods/test/setupssl.py b/irods/test/setupssl.py index 3d3c20205..d14e682c0 100755 --- a/irods/test/setupssl.py +++ b/irods/test/setupssl.py @@ -60,7 +60,7 @@ def create_ssl_dir( # https://www.openssl.org/docs/man1.0.2/man1/dhparam.html#:~:text=DH%20parameter%20generation%20with%20the,that%20may%20be%20possible%20otherwise. if use_strong_primes_for_dh_generation: dhparam_generation_command = ( - "openssl dhparam -2 -out dhparams.pem" + "openssl dhparam -2 -out dhparams.pem 2048" ) else: dhparam_generation_command = ( From 4b1f202115c478779830f7f592cb623b6301b135 Mon Sep 17 00:00:00 2001 From: d-w-moore Date: Fri, 24 Oct 2025 14:21:43 -0400 Subject: [PATCH 2/2] [_775] add test for pam_password secrets file generation prc_write_irodsA.py invokes a function in the irods.client_init module which adapts to iRODS server version, using the new authentication framework in iRODS 4.3+ and legacy auth in previous versions. --- irods/test/harness/test_script_parameters | 1 + ...st011_write_pam_password_secrets_file.bats | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100755 irods/test/scripts/test011_write_pam_password_secrets_file.bats diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters index fe2beca54..2149f58c0 100644 --- a/irods/test/harness/test_script_parameters +++ b/irods/test/harness/test_script_parameters @@ -21,6 +21,7 @@ declare -A wrappers=( [test008_prc_write_irodsA_utility_in_native_mode.bats]=../login_auth_test.sh [test009_test_special_characters_in_pam_passwords_auth_framework.bats]=../login_auth_test.sh [test010_issue_362_rogue_chars_in_pam_password.bats]=../login_auth_test.sh + [test011_write_pam_password_secrets_file.bats]=../login_auth_test.sh ) # keys for Image and User refer to the basename after resolution to a wrapper if one is used diff --git a/irods/test/scripts/test011_write_pam_password_secrets_file.bats b/irods/test/scripts/test011_write_pam_password_secrets_file.bats new file mode 100755 index 000000000..f2674eca5 --- /dev/null +++ b/irods/test/scripts/test011_write_pam_password_secrets_file.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +# The tests in this BATS module must be run as a (passwordless) sudo-enabled user. +# It is also required that the python irodsclient be installed under irods' ~/.local environment. + +. $BATS_TEST_DIRNAME/test_support_functions + +# Setup in the wrapper script (../login_auth_test.sh) includes creation of Linux user alissa with login password test123 . + +@test "test_writing_secrets" { + iadmin mkuser alissa rodsuser + + # Make a new environment and pam_password secrets file for iRODS user alissa. + rm -fr .irods/.irodsA + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_user_name"]="alissa"|.["irods_authentication_scheme"]="pam_password"' >$CLIENT_JSON.$$ <$CLIENT_JSON + mv $CLIENT_JSON.$$ $CLIENT_JSON + /pyN/bin/prc_write_irodsA.py --ttl 10 pam_password <<<"test123" + + # Test that iCommands pam_password auth works with the secrets file. + ils ' + + # Test that python irods client pam_password authentication works with the secrets file. + python -c 'import irods.helpers as h; ses=h.make_session(); c=h.home_collection(ses); print(ses.collections.get(c).path)'|\ + grep '/alissa\>' +}