metadata:
  annotations:
    prow.k8s.io/context: ci/prow/e2e-gke-multi-datacenter-parallel
    prow.k8s.io/job: pull-scylla-operator-master-e2e-gke-multi-datacenter-parallel
  creationTimestamp: "2025-10-07T09:55:12Z"
  generation: 11
  labels:
    created-by-prow: "true"
    event-GUID: b63323e0-a363-11f0-9f34-b80273765aae
    prow.k8s.io/build-id: "1975500148146442240"
    prow.k8s.io/context: e2e-gke-multi-datacenter-parallel
    prow.k8s.io/id: 0d8c6737-5180-47ca-ad02-bffdac1b3f4d
    prow.k8s.io/is-optional: "false"
    prow.k8s.io/job: pull-scylla-operator-master-e2e-gke-multi-datacenter-parallel
    prow.k8s.io/refs.base_ref: master
    prow.k8s.io/refs.org: scylladb
    prow.k8s.io/refs.pull: "2975"
    prow.k8s.io/refs.repo: scylla-operator
    prow.k8s.io/type: presubmit
  name: 0d8c6737-5180-47ca-ad02-bffdac1b3f4d
  namespace: prow-workspace
  resourceVersion: "1759837901773087024"
  uid: 730eec27-100c-4cbe-a974-d4120727703f
spec:
  agent: kubernetes
  cluster: default
  context: ci/prow/e2e-gke-multi-datacenter-parallel
  decoration_config:
    gcs_configuration:
      bucket: gs://scylla-operator-prow
      path_strategy: explicit
    gcs_credentials_secret: gcs-credentials
    github_api_endpoints:
    - http://ghproxy.prow.svc
    - https://api.github.com
    github_app_id: "112385"
    github_app_private_key_secret:
      key: cert
      name: github-token
    grace_period: 15m0s
    resources:
      clonerefs:
        requests:
          cpu: 100m
      initupload:
        requests:
          cpu: 100m
      place_entrypoint:
        requests:
          cpu: 100m
      sidecar:
        requests:
          cpu: 100m
    timeout: 4h0m0s
    utility_images:
      clonerefs: us-docker.pkg.dev/k8s-infra-prow/images/clonerefs:v20250522-9b3f5faca
      entrypoint: us-docker.pkg.dev/k8s-infra-prow/images/entrypoint:v20250522-9b3f5faca
      initupload: us-docker.pkg.dev/k8s-infra-prow/images/initupload:v20250522-9b3f5faca
      sidecar: us-docker.pkg.dev/k8s-infra-prow/images/sidecar:v20250522-9b3f5faca
  job: pull-scylla-operator-master-e2e-gke-multi-datacenter-parallel
  namespace: prow-workspace
  pod_spec:
    containers:
    - args:
      - "worker_clusters_names=(\n  \"europe-west1\"\n  \"europe-west3\"\n  \"europe-west4\"\n)\n\n#
        $1 is the worker cluster name.\nfunction storage_bucket_object_name {\n  echo
        \"${POD_UID}-${1}\"\n}\n\nfunction cleanup {\n  touch /tmp/shared/setup.finished\n
        \ kubectl -n ci-clusters delete --wait=false --ignore-not-found kubernetesclusterset/\"${POD_UID}\"\n
        \ for name in \"${worker_clusters_names[@]}\"; do\n    kubectl -n ci-clusters
        delete --wait=false --ignore-not-found storagebucket/\"$( storage_bucket_object_name
        \"${name}\" )\"\n  done\n}\ntrap cleanup EXIT\n\npull_refs_hash=$( sha512sum
        <<< \"${PULL_REFS}\" | cut -c 1-31 )\nSOCI_PR_TAG=\"pr-${pull_refs_hash}\"\nexport
        SOCI_PR_TAG\n\n# Don't create e2e cluster before an image is build, so we
        don't waste infra when it doesn't compile.\nERR_LOG=$( mktemp )\nexport ERR_LOG\ntimeout
        15m bash -xc 'until skopeo inspect --format='\"'\"'{{ printf \"%s@%s\" .Name
        .Digest }}'\"'\"'  docker://\"quay.io/scylladb-dev/ci:scylla-operator-${SOCI_PR_TAG}\"
        2>>\"${ERR_LOG}\"; do sleep 5; done' || ( cat \"${ERR_LOG}\" && exit 2 )\n\nkubectl
        version -o yaml\n\nkubectl -n ci-clusters create -f - <<EOF\napiVersion: ci.scylladb.com/v1alpha1\nkind:
        KubernetesClusterSet\nmetadata:\n  name: \"${POD_UID}\"\nspec:\n  ttlSeconds:
        14400\n  networking:\n    type: Shared\n  clusterTemplates:\n  - metadata:\n
        \     name: europe-west1\n    spec:\n      version: \"1.33\"\n      credentialsSecret:\n
        \       name: \"${POD_UID}-europe-west1\"\n      type: GKE\n      gke:\n        location:
        europe-west1\n        nodePools:\n        - name: workers\n          locations:\n
        \         - europe-west1-b\n          config:\n            machineType: e2-standard-8\n
        \           diskType: pd-standard\n            diskSizeGB: 40\n            labels:\n
        \             pool: workers\n          initialNodeCount: 1\n        - name:
        scylladb\n          locations:\n          - europe-west1-b\n          config:\n
        \           machineType: c2d-standard-8\n            diskType: pd-balanced\n
        \           diskSizeGB: 40\n            localNVMeSSDBlockConfig:\n              localSSDCount:
        1\n            labels:\n              pool: scylladb\n              scylla.scylladb.com/node-type:
        scylla\n            taints:\n            - key: scylla-operator.scylladb.com/dedicated\n
        \             value: scyllaclusters\n              effect: NO_SCHEDULE\n            kubeletConfig:\n
        \             cpuManagerPolicy: static\n          initialNodeCount: 1\n  -
        metadata:\n      name: europe-west3\n    spec:\n      version: \"1.33\"\n
        \     credentialsSecret:\n        name: \"${POD_UID}-europe-west3\"\n      type:
        GKE\n      gke:\n        location: europe-west3\n        nodePools:\n        -
        name: workers\n          locations:\n          - europe-west3-a\n          config:\n
        \           machineType: e2-standard-8\n            diskType: pd-standard\n
        \           diskSizeGB: 40\n            labels:\n              pool: workers\n
        \         initialNodeCount: 1\n        - name: scylladb\n          locations:\n
        \         - europe-west3-a\n          config:\n            machineType: c2d-standard-8\n
        \           diskType: pd-balanced\n            diskSizeGB: 40\n            localNVMeSSDBlockConfig:\n
        \             localSSDCount: 1\n            labels:\n              pool: scylladb\n
        \             scylla.scylladb.com/node-type: scylla\n            taints:\n
        \           - key: scylla-operator.scylladb.com/dedicated\n              value:
        scyllaclusters\n              effect: NO_SCHEDULE\n            kubeletConfig:\n
        \             cpuManagerPolicy: static\n          initialNodeCount: 1\n  -
        metadata:\n      name: europe-west4\n    spec:\n      version: \"1.33\"\n
        \     credentialsSecret:\n        name: \"${POD_UID}-europe-west4\"\n      type:
        GKE\n      gke:\n        location: europe-west4\n        nodePools:\n        -
        name: workers\n          locations:\n          - europe-west4-a\n          config:\n
        \           machineType: e2-standard-8\n            diskType: pd-standard\n
        \           diskSizeGB: 40\n            labels:\n              pool: workers\n
        \         initialNodeCount: 1\n        - name: scylladb\n          locations:\n
        \         - europe-west4-a\n          config:\n            machineType: c2d-standard-8\n
        \           diskType: pd-balanced\n            diskSizeGB: 40\n            localNVMeSSDBlockConfig:\n
        \             localSSDCount: 1\n            labels:\n              pool: scylladb\n
        \             scylla.scylladb.com/node-type: scylla\n            taints:\n
        \           - key: scylla-operator.scylladb.com/dedicated\n              value:
        scyllaclusters\n              effect: NO_SCHEDULE\n            kubeletConfig:\n
        \             cpuManagerPolicy: static\n          initialNodeCount: 1\nEOF\n\ntimeout
        -v 15m bash -c 'until kubectl -n ci-clusters wait --for=condition=Degraded=False
        kubernetesclusterset/\"${POD_UID}\" --timeout=15m && kubectl -n ci-clusters
        wait --for=condition=Progressing=False kubernetesclusterset/\"${POD_UID}\"
        --timeout=15m; do sleep 1; done'\n\n# Returns the kubeconfig path for the
        worker cluster before it is moved to the final location.\n# $1 - name of the
        worker cluster\nfunction get_worker_kubeconfig_init_path {\n  echo \"/tmp/shared/kubeconfigs.init/${1}.kubeconfig\"\n}\n\n#
        Returns the kubeconfig path for the worker cluster after it is moved to the
        final location.\n# $1 - name of the worker cluster\nfunction get_worker_kubeconfig_path
        {\n  echo \"/tmp/shared/kubeconfigs/${1}.kubeconfig\"\n}\n\n# $1 - name of
        the worker cluster\n# $2 - path where the kubeconfig should be created\nfunction
        prepare_worker_kubeconfig {\n  kubectl -n ci-clusters get secret/\"${POD_UID}-${1}\"
        --template='{{ .data.kubeconfig }}' | base64 -d > \"${2}\" \n  kubectl --kubeconfig=\"${2}\"
        config set-context --current --namespace 'default-unexisting-namespace'\n}\n\n#
        Prepare a regional GCS bucket for each worker cluster.\ngcs_sa_credentials_shared_dir=\"/tmp/shared/gcs-service-account-credentials\"\nmkdir
        -p \"${gcs_sa_credentials_shared_dir}\"\ndeclare -A WORKER_OBJECT_STORAGE_BUCKETS
        WORKER_GCS_SERVICE_ACCOUNT_CREDENTIALS_PATHS\nfor name in \"${worker_clusters_names[@]}\";
        do\n  secret_name=\"${POD_UID}-${name}-gcs-service-account\"\n  object_name=\"$(
        storage_bucket_object_name \"${name}\" )\"\n  path=\"${gcs_sa_credentials_shared_dir}/${name}.json\"\n
        \ \n  kubectl -n ci-clusters create -f - <<EOF\napiVersion: ci.scylladb.com/v1alpha1\nkind:
        StorageBucket\nmetadata:\n  name: \"${object_name}\"\nspec:\n  ttlSeconds:
        14400\n  type: GCS\n  gcs:\n    location: \"${name}\" #  We rely on the fact
        that the names match GCS locations.\n  credentialsSecret:\n    name: \"${secret_name}\"\nEOF\n
        \ \n  # Wait for the bucket to be created and ready.\n  timeout -v 5m bash
        -c \"until kubectl -n ci-clusters wait --for=condition=Degraded=False storagebucket/\"${object_name}\"
        --timeout=5m && kubectl -n ci-clusters wait --for=condition=Progressing=False
        storagebucket/\"${object_name}\" --timeout=5m; do sleep 1; done\"\n  \n  #
        Get the bucket name and store it in the associative array.\n  bucket_name=$(kubectl
        -n ci-clusters get storagebuckets/\"${object_name}\" --template='{{ .status.bucketName
        }}')\n  WORKER_OBJECT_STORAGE_BUCKETS[\"${name}\"]=\"${bucket_name}\"\n\n
        \ # Store the GCS service account credentials in a file and keep the path
        in the associative array.\n  kubectl -n ci-clusters get secret/\"${secret_name}\"
        --template='{{ index .data \"gcs-service-account.json\" }}' | base64 -d >
        \"${path}\"\n  WORKER_GCS_SERVICE_ACCOUNT_CREDENTIALS_PATHS[\"${name}\"]=\"${path}\"\n\n
        \ echo \"Created GCS bucket for worker '${name}'. Bucket '${bucket_name}',
        credentials stored under '${path}'.\"\ndone\n\nmkdir /tmp/shared/kubeconfigs.init\ndeclare
        -A WORKER_KUBECONFIGS\nfor name in \"${worker_clusters_names[@]}\"; do\n  prepare_worker_kubeconfig
        \"${name}\" \"$(get_worker_kubeconfig_init_path \"${name}\")\" \n  WORKER_KUBECONFIGS[\"${name}\"]=\"$(get_worker_kubeconfig_path
        \"${name}\")\"\ndone\n\n# Prepare a script that can be sourced in the test
        container to make associative arrays ready.\n# It's done this way because
        we cannot pass associative arrays as environment variables between containers.\n#
        declare -p will output the associative arrays in a way that can be sourced
        in bash.\ndeclare -p WORKER_KUBECONFIGS WORKER_OBJECT_STORAGE_BUCKETS WORKER_GCS_SERVICE_ACCOUNT_CREDENTIALS_PATHS
        > \\\n  /tmp/shared/setup-associative-array-vars.sh\n\n# Signal test setup
        finalisation by moving all kubeconfigs to the final location.\nmv /tmp/shared/kubeconfigs{.init,}\n\nset
        +x\n# TODO: Wait on active signal like updating a file every X seconds\n#
        \      so we can deal with the other container being OOM killed.\necho \"Waiting
        for test to finish...\"\nuntil [[ -f \"/tmp/shared/test.finished\" ]]; do
        sleep 1; done\necho \"Test has finished.\"\nset -x\n"
      command:
      - /usr/bin/bash
      - -euExo
      - pipefail
      - -O
      - inherit_errexit
      - -c
      env:
      - name: POD_UID
        valueFrom:
          fieldRef:
            fieldPath: metadata.uid
      image: quay.io/scylladb/scylla-operator-images:kube-tools
      imagePullPolicy: Always
      name: setup
      resources:
        limits:
          cpu: 100m
          memory: 200Mi
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop:
          - ALL
        seccompProfile:
          type: RuntimeDefault
      volumeMounts:
      - mountPath: /var/run/secrets/kubernetes.io/serviceaccount/
        name: kube-api-access
      - mountPath: /tmp/shared
        name: shared-data
    - args:
      - |
        trap 'touch /tmp/shared/test.finished' EXIT

        set +x
        echo "Waiting for cluster to be provisioned..."
        until [[ -f "${KUBECONFIG}" ]] || [[ -f /tmp/shared/setup.finished ]]; do sleep 1; done
        if [[ -f /tmp/shared/setup.finished ]]; then
          echo "Test setup failed. Exiting."
          exit 1
        fi
        echo "Test setup has finished."
        set -x

        pull_refs_hash=$( sha512sum <<< "${PULL_REFS}" | cut -c 1-31 )
        SOCI_PR_TAG="pr-${pull_refs_hash}"

        # $1 - kubeconfig file path
        function run_sanity_checks {
          kubeconfig="$1"
          echo "Running sanity checks on kubeconfig: ${kubeconfig}"
          kubectl --kubeconfig="${kubeconfig}" config view
          kubectl --kubeconfig="${kubeconfig}" version -o yaml
          kubectl --kubeconfig="${kubeconfig}" cluster-info
        }

        run_sanity_checks "${KUBECONFIG}"

        source /tmp/shared/setup-associative-array-vars.sh
        for name in "${!WORKER_KUBECONFIGS[@]}"; do
          run_sanity_checks "${WORKER_KUBECONFIGS[${name}]}"
        done

        SO_SUITE=scylla-operator/conformance/multi-datacenter-parallel
        export SO_SUITE
        SO_IMAGE=quay.io/scylladb-dev/ci:scylla-operator-${SOCI_PR_TAG}
        export SO_IMAGE
        SO_SCYLLACLUSTER_NODE_SERVICE_TYPE=Headless
        export SO_SCYLLACLUSTER_NODE_SERVICE_TYPE
        SO_SCYLLACLUSTER_NODES_BROADCAST_ADDRESS_TYPE=PodIP
        export SO_SCYLLACLUSTER_NODES_BROADCAST_ADDRESS_TYPE
        SO_SCYLLACLUSTER_CLIENTS_BROADCAST_ADDRESS_TYPE=PodIP
        export SO_SCYLLACLUSTER_CLIENTS_BROADCAST_ADDRESS_TYPE
        SO_E2E_TIMEOUT=90m
        export SO_E2E_TIMEOUT
        timeout --verbose --signal INT --kill-after=130m 120m bash -c '
          # Run both setup-associative-array-vars.sh and run-e2e-gke.sh in the same process to ensure
          # that all the associative arrays are available in the run-e2e-gke.sh script (associative arrays
          # cannot be passed as environment variables).
          source /tmp/shared/setup-associative-array-vars.sh
          source ./hack/.ci/run-e2e-gke.sh
        '
      command:
      - /usr/bin/bash
      - -euExo
      - pipefail
      - -O
      - inherit_errexit
      - -c
      env:
      - name: KUBECONFIG
        value: /tmp/shared/kubeconfigs/europe-west1.kubeconfig
      image: quay.io/scylladb/scylla-operator-images:kube-tools
      imagePullPolicy: Always
      name: test
      resources:
        limits:
          cpu: 100m
          memory: 1200Mi
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop:
          - ALL
        seccompProfile:
          type: RuntimeDefault
      volumeMounts:
      - mountPath: /tmp/shared
        name: shared-data
    serviceAccountName: e2e
    volumes:
    - emptyDir: {}
      name: shared-data
    - name: kube-api-access
      projected:
        defaultMode: 400
        sources:
        - serviceAccountToken:
            path: token
        - configMap:
            items:
            - key: ca.crt
              path: ca.crt
            name: kube-root-ca.crt
        - downwardAPI:
            items:
            - fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
              path: namespace
  prowjob_defaults:
    tenant_id: GlobalDefaultID
  refs:
    base_link: https://github.com/scylladb/scylla-operator/commit/13d313ba52d1e2170f3dea355a7c4da237a41584
    base_ref: master
    base_sha: 13d313ba52d1e2170f3dea355a7c4da237a41584
    org: scylladb
    path_alias: github.com/scylladb/scylla-operator
    pulls:
    - author: rzetelskik
      author_link: https://github.com/rzetelskik
      commit_link: https://github.com/scylladb/scylla-operator/pull/2975/commits/424ddef34627902ae3d93c2a735c5611215ae29a
      head_ref: bootstrap-synchronisation-poc
      link: https://github.com/scylladb/scylla-operator/pull/2975
      number: 2975
      sha: 424ddef34627902ae3d93c2a735c5611215ae29a
      title: '[WIP][DUMMY] Test a PoC approach to ensure all nodes in the cluster
        see all as UP as a prerequisite for a topology change and each step of a rolling
        restart'
    repo: scylla-operator
    repo_link: https://github.com/scylladb/scylla-operator
  report: true
  rerun_command: /test e2e-gke-multi-datacenter-parallel
  type: presubmit
status:
  build_id: "1975500148146442240"
  completionTime: "2025-10-07T11:51:39Z"
  description: Job failed.
  pendingTime: "2025-10-07T09:55:14Z"
  pod_name: 0d8c6737-5180-47ca-ad02-bffdac1b3f4d
  prev_report_states:
    gcsk8sreporter: failure
    gcsreporter: failure
    github-reporter: failure
  startTime: "2025-10-07T09:55:12Z"
  state: failure
  url: https://prow.scylla-operator.scylladb.com/view/gs/scylla-operator-prow/pr-logs/pull/scylladb_scylla-operator/2975/pull-scylla-operator-master-e2e-gke-multi-datacenter-parallel/1975500148146442240