Merge pull request #4943 from Deltik/hotfix/tests/acceptance

Reintroduce automated acceptance tests

.github/workflows/test-acceptance.yml

@@ -0,0 +1,64 @@
+name: Acceptance Tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test-acceptance:
+    strategy:
+      fail-fast: false
+      matrix:
+        operating_system:
+          - image: docker.io/jrei/systemd-ubuntu:20.04
+          - image: docker.io/jrei/systemd-ubuntu:22.04
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Replace Docker with Podman
+        run: sudo apt-get -o Dpkg::Options::="--force-overwrite" install -y podman-docker
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "^1.19"
+
+      - uses: actions/checkout@v3
+
+      - name: Compile SaltStack bootstrap wrapper
+        run: CGO_ENABLED=0 go build -ldflags "-s -w" -o ./salt-bootstrap salt-bootstrap.go
+        working-directory: ./e107_tests/lib/ci/salt/
+
+      - name: Launch test container
+        run: docker run -d -it --rm --name target -v .:/app/ ${{ matrix.operating_system.image }}
+
+      - name: Install SaltStack
+        run: docker exec target /app/e107_tests/lib/ci/salt/salt-bootstrap onedir
+
+      - name: Apply Salt state
+        run: |
+          ln -v -s master minion
+          rm -fv pillars/config.sls
+          touch pillars/config.sls
+          rm -fv pillars/config-sample.sls
+          touch pillars/config-sample.sls
+          ln -v -s -f ../../config.acceptance.ci.yml pillars/config-local.sls
+          docker exec -w /app/e107_tests/lib/ci/salt/ target /bin/sh -c "
+            salt-call -c ./ --id=e107-dev --local state.apply e107-dev ||
+            salt-call -c ./ --id=e107-dev --local state.apply e107-dev
+          "
+        working-directory: ./e107_tests/lib/ci/salt/
+
+      - name: Install test dependencies
+        run: |
+          docker exec -w /app/e107_tests/ -e COMPOSER_ALLOW_SUPERUSER=1 target \
+          composer update --prefer-dist --no-progress
+
+      - name: Download Git submodule dependencies
+        run: git submodule update --init --recursive --remote
+
+      - name: Install the CI test configuration file
+        run: |
+          ln -v -s -f ./lib/ci/config.acceptance.ci.yml ./e107_tests/config.yml
+
+      - name: Run acceptance tests
+        run: docker exec -w /app/e107_tests/ target php ./vendor/bin/codecept run acceptance --steps

.github/workflows/test-unit.yml

@@ -126,7 +126,7 @@ jobs:
       run: git submodule update --init --recursive --remote
 
     - name: Install the CI test configuration file
-      run: cp ./e107_tests/lib/ci/config.ci.yml ./e107_tests/config.yml
+      run: cp ./e107_tests/lib/ci/config.unit.ci.yml ./e107_tests/config.yml
 
     - name: Download Code Climate test reporter
       run: |

e107_tests/lib/ci/Dockerfile

@@ -1,27 +0,0 @@
-FROM ubuntu:20.04
-
-ENV DEBIAN_FRONTEND noninteractive
-
-RUN apt-get update
-RUN apt-get install -y wget gnupg
-RUN wget -O - https://repo.saltstack.com/py3/ubuntu/20.04/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add -
-RUN mkdir -pv /etc/apt/sources.list.d/
-RUN echo 'deb http://repo.saltstack.com/py3/ubuntu/20.04/amd64/latest focal main' |\
-    tee /etc/apt/sources.list.d/saltstack.list
-RUN apt-get update
-RUN apt-get install -y systemd-sysv salt-minion openssh-server rsync
-RUN systemctl disable salt-minion.service
-RUN mkdir -pv /etc/salt/
-
-COPY salt /var/tmp/salt
-COPY config.ci.yml /var/tmp/salt/pillars/config-local.sls
-RUN rm -fv /var/tmp/salt/pillars/config.sls && touch /var/tmp/salt/pillars/config.sls
-RUN rm -fv /var/tmp/salt/pillars/config-sample.sls && touch /var/tmp/salt/pillars/config-sample.sls
-RUN cp -fv /var/tmp/salt/master /etc/salt/minion
-
-WORKDIR /var/tmp/salt
-RUN salt-call -l debug --id=e107-dev --local state.apply e107-dev
-WORKDIR /
-
-VOLUME ["/sys/fs/cgroup"]
-ENTRYPOINT ["/usr/sbin/init"]

e107_tests/lib/ci/config.acceptance.ci.yml

@@ -0,0 +1,15 @@
+---
+deployer: 'sftp'
+url: 'http://localhost/e107/'
+db:
+  host: '127.0.0.1'
+  dbname: 'app'
+  user: 'app'
+  password: 'Database Password for Continuous Integration'
+  populate: true
+fs:
+  host: 'localhost'
+  user: 'www-data'
+  port: '22'
+  password: 'UNIX Password for Continuous Integration'
+  path: '/var/www/html/e107/'

e107_tests/lib/ci/salt/salt-bootstrap.go

@@ -0,0 +1,70 @@
+/*
+ * e107 website system
+ *
+ * Copyright (C) 2008-2022 e107 Inc (e107.org)
+ * Released under the terms and conditions of the
+ * GNU General Public License (http://www.gnu.org/licenses/gpl.txt)
+ *
+ */
+
+package main
+
+import (
+	"crypto/tls"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"sync"
+)
+
+func main() {
+	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	response, err := http.Get("https://bootstrap.saltproject.io")
+	if err != nil {
+		panic(err)
+	}
+
+	args := append([]string{"-s", "--"}, os.Args[1:]...)
+	cmd := exec.Command("/bin/bash", args...)
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		panic(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		panic(err)
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		panic(err)
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, err = io.Copy(stdin, response.Body)
+		if err != nil {
+			panic(err)
+		}
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, _ = io.Copy(os.Stdout, stdout)
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, _ = io.Copy(os.Stderr, stderr)
+	}()
+	err = cmd.Start()
+	if err != nil {
+		panic(err)
+	}
+	err = cmd.Wait()
+	wg.Wait()
+	if err != nil {
+		os.Exit(err.(*exec.ExitError).ExitCode())
+	}
+}

e107_tests/lib/ci/salt/states/e107-dev.sls

@@ -1,9 +1,21 @@
+Install Python module build dependencies:
+  pkg.installed:
+    - pkgs:
+      - build-essential
+      - libmariadb-dev
+
+MySQLdb Python module for SaltStack:
+  pip.installed:
+    - name: mysqlclient
+    - require:
+      - pkg: "Install Python module build dependencies"
+
 Install LAMP stack:
   pkg.installed:
     - pkgs:
       - mariadb-server
-      - python3-mysqldb
       - php
+      - composer
       - libapache2-mod-php
       - php-mysql
       - php-xml
@@ -11,7 +23,35 @@ Install LAMP stack:
       - php-gd
       - php-mbstring
 
-MySQL server configuration file:
+Install SSH:
+  pkg.installed:
+    - pkgs:
+      - openssh-server
+      - sshpass
+
+Start and enable OpenSSH server:
+  service.running:
+    - name: ssh
+    - enable: True
+    - require:
+      - pkg: Install SSH
+
+Allow user logins:
+  service.running:
+    - name: systemd-user-sessions
+
+Allow user logins (alternate):
+  file.absent:
+    - name: /run/nologin
+    - onfail:
+      - service: Allow user logins
+
+Install Composer:
+  cmd.run:
+    - name: 'wget https://getcomposer.org/installer -O - | php -- --install-dir="/usr/local/bin" --filename="composer"'
+    - unless: test -f /usr/local/bin/composer
+
+MariaDB server configuration file:
   file.managed:
     - name: /etc/mysql/mariadb.conf.d/99-overrides.cnf
     - source: salt://files/etc/mysql/mariadb.conf.d/99-overrides.cnf
@@ -20,31 +60,50 @@ MySQL server configuration file:
     - mode: 0644
     - template: jinja
 
-Start and enable MySQL server daemon:
+Start and enable MariaDB server daemon:
   service.running:
-    - name: mysql
+    - name: mariadb
     - enable: True
     - watch:
       - file: /etc/mysql/mariadb.conf.d/99-overrides.cnf
 
-Create MySQL user:
+Create MariaDB remote user:
   mysql_user.present:
-    - name: {{ salt['pillar.get']('db:user') }}
+    - name: {{ salt['pillar.get']('db:user') | yaml_encode }}
     - host: '%'
-    - password: {{ salt['pillar.get']('db:password') }}
+    - password: {{ salt['pillar.get']('db:password') | yaml_encode }}
     - allow_passwordless: True
     - unix_socket: False
 
-Create MySQL database:
+Create MariaDB local user:
-  mysql_database.present:
+  mysql_user.present:
-    - name: {{ salt['pillar.get']('db:dbname') }}
+    - name: {{ salt['pillar.get']('db:user') | yaml_encode }}
+    - host: 'localhost'
+    - password: {{ salt['pillar.get']('db:password') | yaml_encode }}
+    - allow_passwordless: True
+    - unix_socket: False
 
-Create MySQL grants:
+Create MariaDB database:
+  mysql_database.present:
+    - name: {{ salt['pillar.get']('db:dbname') | yaml_encode }}
+
+Create MariaDB grants for the remote user:
   mysql_grants.present:
     - grant: ALL PRIVILEGES
-    - database: {{ salt['pillar.get']('db:dbname') }}.*
+    - database: {{ (salt['pillar.get']('db:dbname') ~ '.*') | yaml_encode }}
-    - user: {{ salt['pillar.get']('db:user') }}
+    - user: {{ salt['pillar.get']('db:user') | yaml_encode }}
     - host: '%'
+    - require:
+      - mysql_user: Create MariaDB remote user
+
+Create MariaDB grants for the local user:
+  mysql_grants.present:
+    - grant: ALL PRIVILEGES
+    - database: {{ (salt['pillar.get']('db:dbname') ~ '.*') | yaml_encode }}
+    - user: {{ salt['pillar.get']('db:user') | yaml_encode }}
+    - host: 'localhost'
+    - require:
+      - mysql_user: Create MariaDB local user
 
 Start and enable the web server:
   service.running:
@@ -55,16 +114,16 @@ Start and enable the web server:
 
 Configure Apache user:
   user.present:
-    - name: {{ salt['pillar.get']('fs:user') }}
+    - name: {{ salt['pillar.get']('fs:user') | yaml_encode }}
-    - password: {{ salt['pillar.get']('fs:password') }}
+    - password: {{ salt['pillar.get']('fs:password') | yaml_encode }}
     - hash_password: True
     - shell: /bin/bash
 
 Ensure docroot has the correct permissions:
   file.directory:
-    - name: {{ salt['pillar.get']('fs:path') }}
+    - name: {{ salt['pillar.get']('fs:path') | yaml_encode }}
-    - user: {{ salt['pillar.get']('fs:user') }}
+    - user: {{ salt['pillar.get']('fs:user') | yaml_encode }}
-    - group: {{ salt['pillar.get']('fs:user') }}
+    - group: {{ salt['pillar.get']('fs:user') | yaml_encode }}
     - recurse:
       - user
       - group