CI/appveyor: run SSH server for tests on GitHub Actions (#607)

No longer rely on DigitalOcean to host the Docker container.

Unfortunately we require a small dispatcher script that has
access to a GitHub access token with scope repo in order to
trigger the daemon workflow on GitHub Actions also for PRs.

This script is hosted by myself for the time being until GitHub
provides a tighter scope to trigger the workflow_dispatch event.

.github/workflows/appveyor.yml

@@ -0,0 +1,67 @@
+# Copyright (c) 2021 Marc Hoersken
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+name: AppVeyor Docker Bridge
+
+on:
+  workflow_dispatch:
+    inputs:
+      ssh_host:
+        required: true
+      ssh_port:
+        required: true
+      ssh_user:
+        required: true
+      ssh_forward:
+        required: true
+      ssh_hostkey:
+        required: true
+      ssh_privkey:
+        required: true
+
+jobs:
+  daemon:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Setup SSH client configuration
+        run: |
+          cd ~
+          mkdir .ssh
+          chmod 0700 .ssh
+          echo "ServerAliveInterval 45" > .ssh/config
+          echo "Host ${{ github.event.inputs.ssh_host }}" >> .ssh/config
+          echo "${{ github.event.inputs.ssh_forward }}" | sed 's/,/\n/g' | sed 's/^/\tRemoteForward /g' >> .ssh/config
+          chmod 0600 .ssh/config
+          cat .ssh/config
+          echo "${{ github.event.inputs.ssh_host }} ${{ github.event.inputs.ssh_hostkey }}" | sed 's/,/\n${{ github.event.inputs.ssh_host }} /g' > .ssh/known_hosts
+          chmod 0600 .ssh/known_hosts
+          cat .ssh/known_hosts
+          echo "${{ github.event.inputs.ssh_privkey }}" | sed 's/,/\n/g' > .ssh/id_rsa
+          chmod 0600 .ssh/id_rsa
+
+      # we sleep explicitly to allow the remote system to kill the sleep process
+      - name: Connect to AppVeyor and sleep
+        run: |
+          ssh -v -p ${{ github.event.inputs.ssh_port }} ${{ github.event.inputs.ssh_user }}@${{ github.event.inputs.ssh_host }} sleep 1h

appveyor.yml

@@ -1,5 +1,6 @@
 # Copyright (c) 2014, Ruslan Baratov
 # Copyright (c) 2014, 2016 Alexander Lamaison
+# Copyright (c) 2020, 2021 Marc Hoersken
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -59,9 +60,6 @@ environment:
       BUILD_SHARED_LIBS: OFF
       CRYPTO_BACKEND: "WinCNG"
 
-  digitalocean_access_token:
-    secure: 8qRitvrj69Xhf0Tmu27xnz5drmL2YhmOJLGpXIkYyTCC0JNtBoXW6fMcF3u4Uj1+pIQ+TjegQOwYimlz0oivKTro3v3EXro+osAMNJG6NKc=
-
 platform:
   - x86
   - x64
@@ -77,8 +75,12 @@ matrix:
       platform: x64
 
 install:
-  - choco install -y docker
-  - choco install -y docker-machine
+  # prepare local SSH server for reverse tunneling from GitHub Actions hosting our docker container
+  - ps: |
+      $env:OPENSSH_SERVER_PORT = Get-Random -Minimum 2000 -Maximum 2300
+      [System.Environment]::SetEnvironmentVariable("OPENSSH_SERVER_PORT", $env:OPENSSH_SERVER_PORT)
+  - ps: .\ci\appveyor\docker-bridge.ps1
+  - choco install -y docker-cli
 
 build_script:
   - ps: if($env:PLATFORM -eq "x64") { $env:CMAKE_GEN_SUFFIX=" Win64" }
@@ -86,23 +88,34 @@ build_script:
   - cmake --build _builds --config "%CONFIGURATION%"
 
 before_test:
-  - set DOCKER_MACHINE_NAME=appveyor-%APPVEYOR_PROJECT_SLUG%-%APPVEYOR_JOB_ID%
-  - ps: if($env:digitalocean_access_token) { echo "Using DigitalOcean for testing." } else { echo "DigitalOcean not available. Skipping testing." }
-  - ps: if($env:digitalocean_access_token) { docker-machine create --driver digitalocean --digitalocean-access-token $($env:digitalocean_access_token) $($env:DOCKER_MACHINE_NAME) }
-  - ps: if($env:digitalocean_access_token) { docker-machine env $($env:DOCKER_MACHINE_NAME) --shell powershell | Invoke-Expression }
+  - ps: |
+      Write-Host "Waiting for SSH connection from GitHub Actions" -NoNewline
+      while((Get-Process -Name "sshd" -ErrorAction SilentlyContinue).Count -eq 1) {
+        Write-Host "." -NoNewline
+        Start-Sleep -Seconds 1
+      }
+      if((Get-Process -Name "sshd" -ErrorAction SilentlyContinue).Count -gt 1) {
+        $env:DOCKER_HOST = "tcp://127.0.0.1:2375"
+        [System.Environment]::SetEnvironmentVariable("DOCKER_HOST", $env:DOCKER_HOST)
+        Write-Host "... ready!"
+      } else {
+        Write-Host "... failed!"
+      }
 
 test_script:
   - ps: cd _builds
-  - ps: if($env:digitalocean_access_token) { ctest -VV -C $($env:CONFIGURATION) --output-on-failure }
-
-after_test:
-  - ps: if($env:digitalocean_access_token) { docker-machine rm -y $($env:DOCKER_MACHINE_NAME) }
+  - ps: ctest -VV -C $($env:CONFIGURATION) --output-on-failure
 
 on_failure:
-  - ps: if($env:digitalocean_access_token) { docker-machine rm -y $($env:DOCKER_MACHINE_NAME) }
   - ps: if(Test-Path _builds/CMakeFiles/CMakeOutput.log) { cat _builds/CMakeFiles/CMakeOutput.log }
   - ps: if(Test-Path _builds/CMakeFiles/CMakeError.log) { cat _builds/CMakeFiles/CMakeError.log }
 
+on_finish:
+  - ps: |
+      Get-Process -Name "sleep" -ErrorAction SilentlyContinue | Stop-Process
+      Start-Sleep -Seconds 3
+      Get-Process -Name "sshd" -ErrorAction SilentlyContinue | Stop-Process
+
 # whitelist branches to avoid testing feature branches twice (as branch and as pull request)
 branches:
     only:

ci/appveyor/docker-bridge.bat

@@ -0,0 +1,10 @@
+@echo off
+
+netsh interface portproxy add v4tov4 listenport=3389 listenaddress=%1 connectport=22 connectaddress=127.0.0.1
+netsh interface portproxy show all
+
+C:\msys64\usr\bin\sh -l -c "/usr/bin/ssh-keygen -b 2048 -t rsa -f auth -q -N '' && mkdir .ssh && mv auth.pub .ssh/authorized_keys"
+C:\msys64\usr\bin\sh -l -c "/usr/bin/ssh-keygen -A"
+C:\msys64\usr\bin\sh -l -c "/usr/bin/sshd"
+
+C:\msys64\usr\bin\sh -l -c '/usr/bin/curl -F "account=%APPVEYOR_ACCOUNT_NAME%" -F "project=%APPVEYOR_PROJECT_SLUG%" -F "buildid=%APPVEYOR_BUILD_VERSION%" -F "base=%APPVEYOR_REPO_BRANCH%" -F "hash=%APPVEYOR_REPO_COMMIT%" -F "repo=%APPVEYOR_REPO_NAME%" -F "ssh_host=%2" -F "ssh_port=%3" -F "ssh_user=`whoami`" -F "ssh_forward=127.0.0.1:%OPENSSH_SERVER_PORT% 127.0.0.1:%OPENSSH_SERVER_PORT%,127.0.0.1:2375 /var/run/docker.sock" -F "ssh_hostkey=`paste -d , /etc/ssh/ssh_host_*_key.pub`" -F "ssh_privkey=`paste -sd , auth`" -s "https://stuff.marc-hoersken.de/libssh2/dispatch.php"'

ci/appveyor/docker-bridge.ps1

@@ -0,0 +1,22 @@
+# Partially copied from https://github.com/appveyor/ci/blob/master/scripts/enable-rdp.ps1
+
+# get current IP
+$ip = (Get-NetIPAddress -AddressFamily IPv4 | Where-Object {$_.InterfaceAlias -like 'ethernet*'}).IPAddress
+$port = 3389
+if($ip.StartsWith('172.24.')) {
+    $port = 33800 + ($ip.split('.')[2] - 16) * 256 + $ip.split('.')[3]
+} elseif ($ip.StartsWith('192.168.') -or $ip.StartsWith('10.240.')) {
+    # new environment - behind NAT
+    $port = 33800 + ($ip.split('.')[2] - 0) * 256 + $ip.split('.')[3]
+} elseif ($ip.StartsWith('10.0.')) {
+    $port = 33800 + ($ip.split('.')[2] - 0) * 256 + $ip.split('.')[3]
+}
+
+# get external IP
+$extip = (New-Object Net.WebClient).DownloadString('https://www.appveyor.com/tools/my-ip.aspx').Trim()
+
+# allow inbound traffic
+New-NetFirewallRule -DisplayName "SSH via RDP port" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 22,3389
+
+# launch remote docker daemon with reverse SSH tunnel
+& .\ci\appveyor\docker-bridge.bat $ip $extip $port

tests/openssh_fixture.c

@@ -149,10 +149,23 @@ static int build_openssh_server_docker_image(void)
                              "openssh_server");
 }
 
+static const char *openssh_server_port(void)
+{
+    return getenv("OPENSSH_SERVER_PORT");
+}
+
 static int start_openssh_server(char **container_id_out)
 {
-    return run_command(container_id_out,
-                       "docker run --detach -P libssh2/openssh_server");
+    const char *container_host_port = openssh_server_port();
+    if(container_host_port != NULL) {
+        return run_command(container_id_out,
+                           "docker run -d -p %s:22 libssh2/openssh_server",
+                           container_host_port);
+    }
+    else {
+        return run_command(container_id_out,
+                           "docker run -d -p 22 libssh2/openssh_server");
+    }
 }
 
 static int stop_openssh_server(char *container_id)