GH-1283: Optimize number of build compilations in CI

.github/workflows/test.yml

@@ -34,28 +34,16 @@ permissions:
   contents: read
 
 env:
+  BUILD_JDK: "17"
   DOCKER_VOLUME_PREFIX: ".docker/"
+  MAVEN: 3.9.16
 
 jobs:
-  ubuntu:
-    name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }}
+  build-java:
+    name: Build Java artifacts
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        jdk: [17, 21, 25]
-        maven: [3.9.16]
-        image: [ubuntu, conda-jni-cdata]
-        include:
-          - image: ubuntu
-            name: "Ubuntu"
-          - image: conda-jni-cdata
-            name: "Conda JNI"
-    env:
-      JDK: ${{ matrix.jdk }}
-      MAVEN: ${{ matrix.maven }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v7
@@ -65,82 +53,202 @@ jobs:
       - name: Cache Docker Volumes
         uses: actions/cache@v6
         with:
-          path: .docker
-          key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml') }}
-          restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}-
-      - name: Execute Docker Build
+          path: .docker/maven-cache
+          key: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }}
+          restore-keys: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-
+      - name: Build without tests
         env:
-          # Enables build caching, but not strictly required
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+          JDK: ${{ env.BUILD_JDK }}
         run: |
           docker compose run \
+            --rm \
             -e CI=true \
             -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
-            ${{ matrix.image }}
+            ubuntu-artifacts
+          sudo chown -R "$(id -u):$(id -g)" .docker
+      - name: Pack reusable artifacts
+        run: |
+          tar -czf java-build.tgz \
+            .docker/java-build \
+            .docker/maven-cache/repository/org/apache/arrow
+      - name: Upload reusable artifacts
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: java-build
+          path: java-build.tgz
+          retention-days: 1
+      - name: Exclude reactor artifacts from Maven dependency cache
+        run: rm -rf .docker/maven-cache/repository/org/apache/arrow
 
-  macos:
-    name: ${{ matrix.arch }} macOS ${{ matrix.macos }} Java JDK ${{ matrix.jdk }}
-    runs-on: macos-${{ matrix.macos }}
+  test-java:
+    name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven 3.9.16', matrix.jdk) }}
+    needs: build-java
+    runs-on: ${{ matrix.os }}
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
+        jdk: [17, 21, 25]
+        os: [ubuntu-latest]
         include:
-          - arch: AArch64
+          - os: ubuntu-latest
+            compose_service: ubuntu-test-prebuilt
+          - name: AArch64 macOS latest Java JDK 17
+            os: macos-latest
             jdk: 17
-            macos: latest
+            compose_service: ''
+          - name: AMD64 Windows Server 2022 Java JDK 17
+            os: windows-latest
+            jdk: 17
+            compose_service: ''
+    env:
+      JDK: ${{ matrix.jdk }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v7
         with:
-          fetch-depth: 0
           submodules: recursive
       - name: Set up Java
+        if: ${{ !matrix.compose_service }}
         uses: actions/setup-java@v6
         with:
           distribution: 'temurin'
           java-version: ${{ matrix.jdk }}
           cache: 'maven'
-      - name: Build
+      - name: Cache Docker Volumes
+        if: ${{ matrix.compose_service }}
+        uses: actions/cache@v6
+        with:
+          path: .docker/maven-cache
+          key: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }}
+          restore-keys: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-
+      - name: Download reusable artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: java-build
+      - name: Restore reusable artifacts
+        shell: bash
+        run: |
+          rm -rf \
+            .docker/java-build \
+            .docker/maven-cache/repository/org/apache/arrow
+          tar -xzf java-build.tgz
+      - name: Restore reusable artifacts for hosted runner
+        if: ${{ !matrix.compose_service }}
+        shell: bash
+        run: |
+          cp -a .docker/java-build/build build
+          rm -rf "${HOME}/.m2/repository/org/apache/arrow"
+          mkdir -p "${HOME}/.m2/repository/org/apache"
+          cp -a .docker/maven-cache/repository/org/apache/arrow "${HOME}/.m2/repository/org/apache/"
+      - name: Test prebuilt artifacts on macOS/Windows
+        if: ${{ !matrix.compose_service }}
         shell: bash
         env:
+          ARROW_JAVA_TEST_PREBUILT: "ON"
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/build.sh . build jni
-      - name: Test
+        run: ci/scripts/test.sh . build jni
+      - name: Test prebuilt artifacts on Ubuntu Docker
+        if: ${{ matrix.compose_service }}
         shell: bash
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/test.sh . build jni
+        run: |
+          docker compose run \
+            --rm \
+            -e CI=true \
+            -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
+            ${{ matrix.compose_service }}
+      - name: Exclude reactor artifacts from hosted Maven dependency cache
+        if: ${{ !matrix.compose_service && always() }}
+        shell: bash
+        run: rm -rf "${HOME}/.m2/repository/org/apache/arrow"
 
-  windows:
-    name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }}
-    runs-on: windows-latest
+  build-cdata:
+    name: Build C Data artifacts
+    runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        jdk: [17]
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v7
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Set up Java
-        uses: actions/setup-java@v6
+      - name: Cache Docker Volumes
+        uses: actions/cache@v6
         with:
-          java-version: ${{ matrix.jdk }}
-          distribution: 'temurin'
-          cache: 'maven'
-      - name: Build
-        shell: bash
+          path: .docker/maven-cache
+          key: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }}
+          restore-keys: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-
+      - name: Build C Data without tests
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/build.sh . build jni
-      - name: Test
-        shell: bash
+          JDK: ${{ env.BUILD_JDK }}
+        run: |
+          docker compose run \
+            --rm \
+            -e CI=true \
+            -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
+            cdata-artifacts
+          sudo chown -R "$(id -u):$(id -g)" .docker
+      - name: Pack reusable C Data artifacts
+        run: |
+          tar -czf cdata-build.tgz \
+            .docker/cdata-build \
+            .docker/cdata-jni-dist \
+            .docker/maven-cache/repository/org/apache/arrow
+      - name: Upload reusable C Data artifacts
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: cdata-build
+          path: cdata-build.tgz
+          retention-days: 1
+      - name: Exclude reactor artifacts from Maven dependency cache
+        run: rm -rf .docker/maven-cache/repository/org/apache/arrow
+
+  test-cdata:
+    name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven 3.9.16
+    needs: build-cdata
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [17, 21, 25]
+    env:
+      JDK: ${{ matrix.jdk }}
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v7
+        with:
+          submodules: recursive
+      - name: Cache Docker Volumes
+        uses: actions/cache@v6
+        with:
+          path: .docker/maven-cache
+          key: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }}
+          restore-keys: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-
+      - name: Download reusable C Data artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: cdata-build
+      - name: Restore reusable C Data artifacts
+        run: |
+          rm -rf \
+            .docker/cdata-build \
+            .docker/cdata-jni-dist \
+            .docker/maven-cache/repository/org/apache/arrow
+          tar -xzf cdata-build.tgz
+      - name: Test C Data without compiler lifecycle
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-        run: ci/scripts/test.sh . build jni
+        run: |
+          docker compose run \
+            --rm \
+            -e CI=true \
+            -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \
+            cdata-test-prebuilt

ci/scripts/build.sh

@@ -46,7 +46,7 @@ cp -r "${source_dir}/dev" "${build_dir}"
 # crawl back up to the top.  GNU realpath has --relative-to but this does not
 # work on macOS
 
-poms=$(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml)
+poms=$(find "${source_dir}" \( -path "${source_dir}"/build -o -path "${source_dir}"/.docker \) -prune -o -type f -name pom.xml -print)
 if [[ "$OSTYPE" == "darwin"* ]]; then
   poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))")
 else

ci/scripts/test.sh

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -ex
+set -exo pipefail
 
 if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then
   exit
@@ -31,14 +31,72 @@ if [ -d "${java_jni_dist_dir}" ]; then
   java_jni_dist_dir="$(cd "${java_jni_dist_dir}" && pwd)"
 fi
 
-mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
-# Use `2 * ncores` threads
-mvn="${mvn} -T 2C"
-mvn="${mvn} -Denforcer.skip=true"
+mvn=(
+  mvn
+  -B
+  -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
+  -T
+  2C
+  -Denforcer.skip=true
+)
+
+run_tests() {
+  if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then
+    run_prebuilt_tests "${@}" surefire:test
+  else
+    "${@}" test
+  fi
+}
+
+run_prebuilt_tests() {
+  local log
+  log=$(mktemp)
+
+  "${@}" -Ddevelocity.cache.local.enabled=false | tee "${log}"
+
+  if grep -E "Compiling [0-9]+ source files?" "${log}"; then
+    echo "Unexpected compilation occurred while running prebuilt tests."
+    exit 1
+  fi
+
+  if ! grep -q "Tests run:" "${log}"; then
+    echo "No surefire test summary found; tests may have been skipped."
+    exit 1
+  fi
+
+  rm -f "${log}"
+}
 
 pushd "${build_dir}"
 
-${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" test
+if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then
+  if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then
+    run_prebuilt_tests \
+      "${mvn[@]}" \
+      -Darrow.test.dataRoot="${source_dir}/testing/data" \
+      -pl "!vector" \
+      surefire:test
+    # Direct Surefire skips Vector's lifecycle-bound allocator test passes.
+    run_prebuilt_tests \
+      "${mvn[@]}" \
+      -Darrow.test.dataRoot="${source_dir}/testing/data" \
+      -pl vector \
+      org.apache.maven.plugins:maven-surefire-plugin:test@default-test
+    run_prebuilt_tests \
+      "${mvn[@]}" \
+      -Darrow.test.dataRoot="${source_dir}/testing/data" \
+      -pl vector \
+      org.apache.maven.plugins:maven-surefire-plugin:test@run-unsafe
+    run_prebuilt_tests \
+      "${mvn[@]}" \
+      -pl memory/memory-core \
+      org.apache.maven.plugins:maven-surefire-plugin:test@opens-tests
+  else
+    run_tests \
+      "${mvn[@]}" \
+      -Darrow.test.dataRoot="${source_dir}/testing/data"
+  fi
+fi
 
 projects=()
 if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
@@ -47,7 +105,8 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then
   projects+=(gandiva)
 fi
 if [ "${#projects[@]}" -gt 0 ]; then
-  ${mvn} test \
+  run_tests \
+    "${mvn[@]}" \
     -Parrow-jni \
     -pl "$(
       IFS=,
@@ -57,7 +116,11 @@ if [ "${#projects[@]}" -gt 0 ]; then
 fi
 
 if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then
-  ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}"
+  run_tests \
+    "${mvn[@]}" \
+    -Parrow-c-data \
+    -pl c \
+    -Darrow.c.jni.dist.dir="${java_jni_dist_dir}"
 fi
 
 popd