.gitlab-ci.yml 2.71 KB
Newer Older
Swaroop Vattam's avatar
Swaroop Vattam committed
1 2 3 4
variables:
  DATA_SUPPLY_COMMIT: ed9c9ebb878f1a3cf7508e8761192c9e8084e25b

# Run full validation of both the repository and datasets, but just in the repository itself.
Mitar's avatar
Mitar committed
5 6 7 8 9 10 11
test:
  stage: build

  image: registry.gitlab.com/datadrivendiscovery/images/core:ubuntu-bionic-python36-devel

  variables:
    GIT_STRATEGY: clone
Swaroop Vattam's avatar
Swaroop Vattam committed
12 13 14
    # We on purpose do not fetch submodules so that we validate just the current repository.
    # This assumes submodules are validated in their own repositories.
    GIT_SUBMODULE_STRATEGY: none
Mitar's avatar
Mitar committed
15 16

  before_script:
Swaroop Vattam's avatar
Swaroop Vattam committed
17
    - "[ ! -f $(git rev-parse --git-dir)/shallow ] || ( echo 'Repository is shallow.' && exit 1 )"
Mitar's avatar
Mitar committed
18 19 20
    - git lfs fetch --all
    - pip3 install cerberus==1.3.1 deep_dircmp==0.1.0
    - git clone --recursive https://gitlab.com/datadrivendiscovery/data-supply.git
Swaroop Vattam's avatar
Swaroop Vattam committed
21
    - git -C data-supply checkout "${DATA_SUPPLY_COMMIT}"
Mitar's avatar
Mitar committed
22 23 24 25 26 27 28 29 30 31 32 33 34

  script:
    - |
      set -o errexit
      echo "Checking repository."
      ./git-check.sh
      echo "Updating digests."
      ./update-digest.py
      echo "Validating datasets."
      ./validate.py
      if [ "${CI_COMMIT_REF_NAME}" = master ]; then
        if [ -n "${GIT_ACCESS_USER}" -a -n "${GIT_ACCESS_TOKEN}" ]; then
          echo "Pushing updated digests."
Mitar's avatar
Mitar committed
35
          git remote set-url --push origin "https://${GIT_ACCESS_USER}:${GIT_ACCESS_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git"
Mitar's avatar
Mitar committed
36 37 38 39 40 41 42 43 44 45 46 47 48 49
          git config --local user.email noreply@datadrivendiscovery.org
          git config --local user.name "D3M CI"
          if ! git diff --quiet ; then
            git commit -a -m "Generated by CI." -m "Source commmit: ${CI_COMMIT_SHA}" -m "[skip ci]"
            if [ "${GIT_DEBUG}" = 1 ]; then
              GIT_TRACE=1 GIT_TRANSFER_TRACE=1 GIT_CURL_VERBOSE=1 git push origin HEAD:refs/heads/master
            else
              git push origin HEAD:refs/heads/master &>/dev/null
            fi
          else
            echo "Nothing changed."
          fi
        fi
      fi
Swaroop Vattam's avatar
Swaroop Vattam committed
50
    - echo "SUCCESS"
Swaroop Vattam's avatar
Swaroop Vattam committed
51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79

# Run just validator of datasets, but also on git submodule.
test_recursive:
  stage: build

  image: registry.gitlab.com/datadrivendiscovery/images/core:ubuntu-bionic-python36-devel

  variables:
    GIT_STRATEGY: clone
    GIT_SUBMODULE_STRATEGY: recursive

  before_script:
    - "[ ! -f $(git rev-parse --git-dir)/shallow ] || ( echo 'Repository is shallow.' && exit 1 )"
    - git lfs fetch --all
    - git submodule foreach --recursive "git lfs fetch --all"
    - pip3 install cerberus==1.3.1 deep_dircmp==0.1.0
    - git clone --recursive https://gitlab.com/datadrivendiscovery/data-supply.git
    - git -C data-supply checkout "${DATA_SUPPLY_COMMIT}"

  script:
    - |
      set -o errexit
      echo "Validating datasets."
      ./validate.py
    - echo "SUCCESS"

  only:
    refs:
      - master