From 1c1e3c73fb88544f87dcfe28fb3731784ea43044 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 11:21:46 -0400 Subject: [PATCH 001/165] first draft of opensearch platform release --- ci/create-bosh-test-release.yml | 4 +- ci/e2e-tests.yml | 4 +- ci/pipeline.yml | 1480 ++---------------- ci/reset-aide.yml | 4 +- ci/run-unit-tests.yml | 4 +- ci/terraform-secrets.sh | 2 +- ci/terraform-secrets.yml | 4 +- ci/update-test-user-passwords.sh | 2 +- ci/upload-audit-events-s3.yml | 4 +- opensearch-base.yml | 328 +--- opensearch-scaling-development.yml | 34 +- opensearch-scaling-production.yml | 175 --- opensearch-scaling-staging.yml | 173 -- opensearch-scaling-testing.yml | 158 -- opsfiles/add-keystore-credentials-errand.yml | 40 - opsfiles/enable-auth-proxy-route-dev.yml | 2 +- opsfiles/enable-dashboard-dns.yml | 2 +- opsfiles/enable-node-tls.yml | 93 -- opsfiles/enable-proxy-auth.yml | 19 - opsfiles/enable-syslog.yml | 24 +- opsfiles/opensearch-notification.yml | 36 - opsfiles/test-name.yml | 2 +- prod-opensearch-ingestors.yml | 74 - scripts/download-e2e-ci-results.sh | 4 +- 24 files changed, 150 insertions(+), 2522 deletions(-) delete mode 100644 opensearch-scaling-production.yml delete mode 100644 opensearch-scaling-staging.yml delete mode 100644 opensearch-scaling-testing.yml delete mode 100644 prod-opensearch-ingestors.yml diff --git a/ci/create-bosh-test-release.yml b/ci/create-bosh-test-release.yml index 702ba73..a5a0293 100644 --- a/ci/create-bosh-test-release.yml +++ b/ci/create-bosh-test-release.yml @@ -12,13 +12,13 @@ image_resource: inputs: - name: release-git-repo - - name: deploy-logs-opensearch-config + - name: deploy-logs-platform-config - name: final-builds-dir-tarball - name: releases-dir-tarball outputs: - name: finalized-release run: - path: deploy-logs-opensearch-config/ci/create-bosh-test-release.sh + path: deploy-logs-platform-config/ci/create-bosh-test-release.sh params: PRIVATE_YML_CONTENT: diff --git a/ci/e2e-tests.yml b/ci/e2e-tests.yml index ecd475c..759f99f 100644 --- a/ci/e2e-tests.yml +++ b/ci/e2e-tests.yml @@ -1,12 +1,12 @@ platform: linux inputs: -- name: deploy-logs-opensearch-config +- name: deploy-logs-platform-config run: path: sh args: - -exc - | - cd deploy-logs-opensearch-config + cd deploy-logs-platform-config ./scripts/e2e.sh diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 4420340..814320a 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -1,1027 +1,127 @@ --- groups: - - name: all - jobs: - - set-self - - audit-to-s3-development - - audit-to-s3-staging - - audit-to-s3-production - - build-opensearch-release - - deploy-opensearch-development - - upload-dashboards-objects-development - - smoke-tests-development - - e2e-tests-development - - tenant-development - - deploy-opensearch-staging - - upload-dashboards-objects-staging - - smoke-tests-staging - - e2e-tests-staging - - tenant-staging - - plan-opensearch-production - - deploy-opensearch-production - - upload-dashboards-objects-production - - smoke-tests-production - - e2e-tests-production - - tenant-production - - run-tests - - name: tasks - jobs: - - audit-to-s3-development - - audit-to-s3-staging - - audit-to-s3-production - - build-opensearch-release - - name: update-test-users - jobs: - - update-test-users-dev - - update-test-users-staging - - update-test-users-production - - name: test - jobs: - - build-opensearch-test-release - - deploy-opensearch-test - - tenant-test - - upload-dashboards-objects-test - name: development jobs: - - deploy-opensearch-development - - upload-dashboards-objects-development - - smoke-tests-development - - e2e-tests-development - - tenant-development - - audit-to-s3-development - - name: staging - jobs: - - deploy-opensearch-staging - - upload-dashboards-objects-staging - - smoke-tests-staging - - e2e-tests-staging - - tenant-staging - - audit-to-s3-staging - - name: production - jobs: - - plan-opensearch-production - - deploy-opensearch-production - - upload-dashboards-objects-production - - smoke-tests-production - - e2e-tests-production - - tenant-production - - audit-to-s3-production - - name: reset-aide - jobs: - - reset-aide-opensearch-production - - -jobs: - - name: set-self - plan: - - get: deploy-logs-opensearch-config - trigger: true - - set_pipeline: self - file: deploy-logs-opensearch-config/ci/pipeline.yml - - - name: build-opensearch-release - plan: - - in_parallel: - - get: release-git-repo - resource: opensearch-release-git-repo - trigger: true - - get: pipeline-tasks - - get: final-builds-dir-tarball - resource: opensearch-final-builds-dir-tarball - - get: releases-dir-tarball - resource: opensearch-releases-dir-tarball - - get: general-task - - task: run-tests - image: general-task - config: - platform: linux - inputs: - - name: release-git-repo - run: - path: sh - args: - - -exc - - | - cd release-git-repo - bundle install - # run release template unit tests - bundle exec rspec - - task: finalize-release - file: pipeline-tasks/finalize-bosh-release.yml - tags: [iaas] - params: - AWS_DEFAULT_REGION: ((aws_default_region)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - PRIVATE_YML_CONTENT: |- - --- - blobstore: - options: - region: ((s3-bosh-blobstore-info.region)) - bucket_name: ((s3-bosh-blobstore-info.bucket_name)) - credentials_source: ((s3-bosh-blobstore-info.credentials_source)) - server_side_encryption: ((s3-bosh-blobstore-info.server_side_encryption)) - - in_parallel: - - put: opensearch-release - tags: [iaas] - params: - file: finalized-release/opensearch-*.tgz - - put: opensearch-final-builds-dir-tarball - tags: [iaas] - params: - file: finalized-release/final-builds-dir-opensearch.tgz - - put: opensearch-releases-dir-tarball - tags: [iaas] - params: - file: finalized-release/releases-dir-opensearch.tgz - on_failure: - put: slack - params: &slack-failure-params - text: | - :x: FAILED to build OpenSearch BOSH release - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - channel: ((slack-channel-customer-success)) - username: ((slack-username)) - icon_url: ((slack-icon-url)) - - - name: build-opensearch-test-release - plan: - - in_parallel: - - get: release-git-repo - resource: opensearch-test-release-git-repo - trigger: true - - get: deploy-logs-opensearch-test-config - - get: final-builds-dir-tarball - resource: opensearch-final-builds-dir-tarball - - get: releases-dir-tarball - resource: opensearch-releases-dir-tarball - - get: general-task - - task: run-tests - image: general-task - config: - platform: linux - inputs: - - name: release-git-repo - run: - path: sh - args: - - -exc - - | - cd release-git-repo - bundle install - # run release template unit tests - bundle exec rspec - - task: create-test-release - input_mapping: - deploy-logs-opensearch-config: deploy-logs-opensearch-test-config - file: deploy-logs-opensearch-test-config/ci/create-bosh-test-release.yml - tags: [iaas] - params: - AWS_DEFAULT_REGION: ((aws_default_region)) - AWS_ACCESS_KEY_ID: ((aws_access_key_id)) - AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) - PRIVATE_YML_CONTENT: |- - --- - blobstore: - options: - region: ((s3-bosh-blobstore-info.region)) - bucket_name: ((s3-bosh-blobstore-info.bucket_name)) - credentials_source: ((s3-bosh-blobstore-info.credentials_source)) - server_side_encryption: ((s3-bosh-blobstore-info.server_side_encryption)) - - put: opensearch-test-release - tags: [iaas] - params: - file: finalized-release/opensearch-test-*.tgz - - - name: run-tests - plan: - - get: general-task - - get: deploy-logs-opensearch-config - trigger: true - passed: [set-self] - - task: run-unit-tests - image: general-task - file: deploy-logs-opensearch-config/ci/run-unit-tests.yml - - - name: audit-to-s3-development - plan: - - get: general-task - - get: ten-minute-timer - trigger: true - - get: deploy-logs-opensearch-config - - get: pipeline-tasks - - task: transfer-audit-logs-to-s3 - image: general-task - file: deploy-logs-opensearch-config/ci/upload-audit-events-s3.yml - params: - BUCKET: ((opensearch-audit-bucket-development)) - UAA_API_URL: ((uaa-url-development)) - UAA_CLIENT_ID: ((uaa-read-only-client-username)) - UAA_CLIENT_SECRET: ((uaa-read-only-client-secret-development)) - CF_API_URL: ((dev-cf-api-url)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_ACCESS_KEY_ID: ((s3_ingestor_aws_access_key_id_development)) - AWS_SECRET_ACCESS_KEY: ((s3_ingestor_aws_secret_access_key_development)) - DEFAULT_TIME: 10 - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Failed to upload audit to s3 in development - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: audit-to-s3-staging - plan: - - get: general-task - - get: ten-minute-timer - trigger: true - - get: deploy-logs-opensearch-config - - get: pipeline-tasks - - task: transfer-audit-logs-to-s3 - image: general-task - file: deploy-logs-opensearch-config/ci/upload-audit-events-s3.yml - params: - BUCKET: ((opensearch-audit-bucket-staging)) - CF_API_URL: ((staging-cf-api-url)) - UAA_API_URL: ((uaa-url-staging)) - UAA_CLIENT_ID: ((uaa-read-only-client-username)) - UAA_CLIENT_SECRET: ((uaa-read-only-client-secret-staging)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_ACCESS_KEY_ID: ((s3_ingestor_aws_access_key_id_staging)) - AWS_SECRET_ACCESS_KEY: ((s3_ingestor_aws_secret_access_key_staging)) - DEFAULT_TIME: 10 - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Failed to upload audit to s3 in staging - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: audit-to-s3-production - plan: - - get: general-task - - get: ten-minute-timer - trigger: true - - get: deploy-logs-opensearch-config - - get: pipeline-tasks - - task: transfer-audit-logs-to-s3 - image: general-task - file: deploy-logs-opensearch-config/ci/upload-audit-events-s3.yml - params: - BUCKET: ((opensearch-audit-bucket-production)) - CF_API_URL: ((cf-api-url-production)) - UAA_API_URL: ((uaa-url-production)) - UAA_CLIENT_ID: ((uaa-read-only-client-username)) - UAA_CLIENT_SECRET: ((uaa-read-only-client-secret-production)) - AWS_DEFAULT_REGION: ((aws-region)) - AWS_ACCESS_KEY_ID: ((s3_ingestor_aws_access_key_id_production)) - AWS_SECRET_ACCESS_KEY: ((s3_ingestor_aws_secret_access_key_production)) - DEFAULT_TIME: 10 - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Failed to upload audit to s3 in production - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: update-test-users-dev - serial: true - plan: - - get: deploy-logs-opensearch-config - params: { depth: 1 } - - get: general-task - - get: weekly - trigger: true - - task: update-test-user-credentials - image: general-task - config: - inputs: - - name: deploy-logs-opensearch-config - platform: linux - run: - path: deploy-logs-opensearch-config/ci/update-test-user-passwords.sh - params: - BOSH_DIRECTOR_NAME: development - UAA_API_URL: ((uaa-url-development)) - UAA_CLIENT_ID: ((uaa-client-id-development)) - UAA_CLIENT_SECRET: ((uaa-client-secret-development)) - TEST_USERS_CREDENTIAL_USERNAME_MAP: ((dev-test-users-credential-username-map)) - CREDHUB_CA_CERT: ((master-bosh-ca.certificate)) - CREDHUB_CLIENT: ((opensearch-ci-credhub-client-id)) - CREDHUB_SECRET: ((opensearch-ci-credhub-client-secret)) - CREDHUB_SERVER: ((credhub-api-server)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Failed to update test users - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: update-test-users-staging - serial: true - plan: - - get: deploy-logs-opensearch-config - params: { depth: 1 } - - get: general-task - - get: weekly - trigger: true - - task: update-test-user-credentials - image: general-task - config: - inputs: - - name: deploy-logs-opensearch-config - platform: linux - run: - path: deploy-logs-opensearch-config/ci/update-test-user-passwords.sh - params: - BOSH_DIRECTOR_NAME: staging - UAA_API_URL: ((uaa-url-staging)) - UAA_CLIENT_ID: ((uaa-client-id-staging)) - UAA_CLIENT_SECRET: ((uaa-client-secret-staging)) - TEST_USERS_CREDENTIAL_USERNAME_MAP: ((staging-test-users-credential-username-map)) - CREDHUB_CA_CERT: ((master-bosh-ca.certificate)) - CREDHUB_CLIENT: ((opensearch-ci-credhub-client-id)) - CREDHUB_SECRET: ((opensearch-ci-credhub-client-secret)) - CREDHUB_SERVER: ((credhub-api-server)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Failed to update test users - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: update-test-users-production - serial: true - plan: - - get: deploy-logs-opensearch-config - params: { depth: 1 } - - get: general-task - - get: weekly - trigger: true - - task: update-test-user-credentials - image: general-task - config: - inputs: - - name: deploy-logs-opensearch-config - platform: linux - run: - path: deploy-logs-opensearch-config/ci/update-test-user-passwords.sh - params: - BOSH_DIRECTOR_NAME: production - UAA_API_URL: ((uaa-url-production)) - UAA_CLIENT_ID: ((uaa-client-id-production)) - UAA_CLIENT_SECRET: ((uaa-client-secret-production)) - TEST_USERS_CREDENTIAL_USERNAME_MAP: ((production-test-users-credential-username-map)) - CREDHUB_CA_CERT: ((master-bosh-ca.certificate)) - CREDHUB_CLIENT: ((opensearch-ci-credhub-client-id)) - CREDHUB_SECRET: ((opensearch-ci-credhub-client-secret)) - CREDHUB_SERVER: ((credhub-api-server)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Failed to update test users - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: deploy-opensearch-test - plan: - - in_parallel: - - get: pipeline-tasks - - get: deploy-logs-opensearch-test-config - - get: opensearch-test-release - trigger: true - - get: opensearch-stemcell-jammy - trigger: true - - get: terraform-yaml - resource: terraform-yaml-development - - get: general-task - - task: opensearch-manifest - image: general-task - config: - platform: linux - inputs: - - name: deploy-logs-opensearch-test-config - - name: terraform-yaml - run: - path: sh - args: - - -exc - - | - bosh int deploy-logs-opensearch-test-config/opensearch-base.yml \ - -o deploy-logs-opensearch-test-config/opensearch-scaling-testing.yml \ - > opensearch-manifest/manifest.yml - outputs: - - name: opensearch-manifest - - task: terraform-secrets - image: general-task - input_mapping: - deploy-logs-opensearch-config: deploy-logs-opensearch-test-config - file: deploy-logs-opensearch-test-config/ci/terraform-secrets.yml - - put: opensearch-test-deployment - params: - manifest: opensearch-manifest/manifest.yml - releases: - - opensearch-test-release/*.tgz - stemcells: - - opensearch-stemcell-jammy/*.tgz - ops_files: - - deploy-logs-opensearch-test-config/opsfiles/enable-node-tls.yml - - deploy-logs-opensearch-test-config/opsfiles/enable-dashboard-dns.yml - - deploy-logs-opensearch-test-config/opsfiles/enable-dashboards-tls.yml - - deploy-logs-opensearch-test-config/opsfiles/enable-proxy-auth.yml - - deploy-logs-opensearch-test-config/opsfiles/enable-auth-proxy-route-test.yml - - deploy-logs-opensearch-test-config/opsfiles/enable-syslog.yml - - deploy-logs-opensearch-test-config/opsfiles/cf-development.yml - - deploy-logs-opensearch-test-config/opsfiles/opensearch-notification.yml - - deploy-logs-opensearch-test-config/opsfiles/test-name.yml - - deploy-logs-opensearch-test-config/opsfiles/add-keystore-credentials-errand.yml - vars_files: - - terraform-secrets/terraform.yml - - - name: tenant-test - plan: - - in_parallel: - - get: general-task - - get: pipeline-tasks - - get: deploy-logs-opensearch-test-config - passed: [deploy-opensearch-test] - trigger: true - - get: opensearch-test-deployment - trigger: true - passed: [deploy-opensearch-test] - - task: create-tenants - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_development_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_development_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch-test - BOSH_ERRAND: upload_tenant - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - - - name: upload-dashboards-objects-test - plan: - - in_parallel: - - get: pipeline-tasks - - get: opensearch-release - - get: opensearch-stemcell-jammy - - get: deploy-logs-opensearch-test-config - passed: [deploy-opensearch-test] - trigger: true - - get: opensearch-test-deployment - passed: [deploy-opensearch-test] - trigger: true - - task: upload-dashboards-objects - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_development_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_development_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch-test - BOSH_ERRAND: upload-dashboards-objects - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - - - name: deploy-opensearch-development - serial_groups: [bosh-development] - plan: - - in_parallel: - - get: pipeline-tasks - - get: deploy-logs-opensearch-config - trigger: true - passed: [run-tests] - - get: opensearch-release - trigger: true - - get: opensearch-stemcell-jammy - trigger: true - - get: general-task - - get: terraform-yaml - resource: terraform-yaml-development - trigger: true - - task: opensearch-manifest - image: general-task - config: - platform: linux - inputs: - - name: deploy-logs-opensearch-config - run: - path: sh - args: - - -exc - - | - bosh int deploy-logs-opensearch-config/opensearch-base.yml \ - -o deploy-logs-opensearch-config/opensearch-scaling-development.yml \ - > opensearch-manifest/manifest.yml - outputs: - - name: opensearch-manifest - - task: terraform-secrets - image: general-task - file: deploy-logs-opensearch-config/ci/terraform-secrets.yml - - put: opensearch-development-deployment - params: - manifest: opensearch-manifest/manifest.yml - releases: - - opensearch-release/*.tgz - stemcells: - - opensearch-stemcell-jammy/*.tgz - ops_files: - - deploy-logs-opensearch-config/opsfiles/enable-node-tls.yml - - deploy-logs-opensearch-config/opsfiles/enable-dashboard-dns.yml - - deploy-logs-opensearch-config/opsfiles/enable-dashboards-tls.yml - - deploy-logs-opensearch-config/opsfiles/enable-proxy-auth.yml - - deploy-logs-opensearch-config/opsfiles/enable-auth-proxy-route-dev.yml - - deploy-logs-opensearch-config/opsfiles/enable-syslog.yml - - deploy-logs-opensearch-config/opsfiles/cf-development.yml - - deploy-logs-opensearch-config/opsfiles/opensearch-notification.yml - - deploy-logs-opensearch-config/opsfiles/add-keystore-credentials-errand.yml - vars_files: - - terraform-secrets/terraform.yml - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: FAILED to deploy logs-OpenSearch in development - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: smoke-tests-development - serial_groups: [bosh-development] - plan: - - in_parallel: - - get: tests-timer - trigger: true - - get: pipeline-tasks - - get: opensearch-release - trigger: true - passed: [deploy-opensearch-development] - - get: opensearch-stemcell-jammy - trigger: true - passed: [deploy-opensearch-development] - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-development] - trigger: true - - get: opensearch-development-deployment - trigger: true - passed: [deploy-opensearch-development] - - task: smoke-tests - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_development_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_development_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: smoke_tests - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Smoke tests for OpenSearch in development FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: e2e-tests-development - serial_groups: [bosh-development] - plan: - - in_parallel: - - get: pipeline-tasks - - get: opensearch-release - trigger: true - passed: [deploy-opensearch-development] - - get: opensearch-stemcell-jammy - trigger: true - passed: [deploy-opensearch-development] - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-development] - trigger: true - - get: opensearch-development-deployment - trigger: true - passed: [deploy-opensearch-development] - - get: one-hour-timer - trigger: true - - get: playwright-python - - get: general-task - - task: provision-test-user-cf-access - image: general-task - config: - platform: linux - inputs: - - name: deploy-logs-opensearch-config - run: - path: deploy-logs-opensearch-config/ci/provision-test-user-cf-access.sh - params: - CF_API_URL: ((dev-cf-api-url)) - CF_USERNAME: ((development-cf-username)) - CF_PASSWORD: ((development-cf-password)) - TEST_USER_1_USERNAME: ((development-test-user-1-username)) - TEST_USER_2_USERNAME: ((development-test-user-2-username)) - TEST_USER_3_USERNAME: ((development-test-user-3-username)) - TEST_USER_4_USERNAME: ((development-test-user-4-username)) - CF_ORG_1_NAME: ((test-org-1)) - CF_ORG_2_NAME: ((test-org-2)) - CF_ORG_3_NAME: ((test-org-3)) - - task: e2e-tests - image: playwright-python - file: deploy-logs-opensearch-config/ci/e2e-tests.yml - params: - AUTH_PROXY_URL: ((development-auth-proxy-url)) - UAA_BASE_URL: ((development-uaa-base-url)) - CF_ORG_1_NAME: ((test-org-1)) - CF_ORG_2_NAME: ((test-org-2)) - CF_ORG_3_NAME: ((test-org-3)) - - TEST_USER_1_USERNAME: ((development-test-user-1-username)) - TEST_USER_1_PASSWORD: ((development-test-user-1-password)) - TEST_USER_1_TOTP_SEED: ((development-test-user-1-totp-seed)) - - TEST_USER_2_USERNAME: ((development-test-user-2-username)) - TEST_USER_2_PASSWORD: ((development-test-user-2-password)) - TEST_USER_2_TOTP_SEED: ((development-test-user-2-totp-seed)) - - TEST_USER_3_USERNAME: ((development-test-user-3-username)) - TEST_USER_3_PASSWORD: ((development-test-user-3-password)) - TEST_USER_3_TOTP_SEED: ((development-test-user-3-totp-seed)) - - TEST_USER_4_USERNAME: ((development-test-user-4-username)) - TEST_USER_4_PASSWORD: ((development-test-user-4-password)) - TEST_USER_4_TOTP_SEED: ((development-test-user-4-totp-seed)) - - SMTP_SENDER_HOST: ((log-alerts-smtp-sender-host)) - SMTP_SENDER_PORT: ((log-alerts-smtp-sender-port)) - SMTP_SENDER_FROM: ((development-log-alerts-smtp-sender-from)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: E2E tests for OpenSearch in development FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: tenant-development - serial_groups: [bosh-development] - plan: - - in_parallel: - - get: general-task - - get: pipeline-tasks - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-development] - trigger: true - - get: opensearch-development-deployment - trigger: true - passed: [deploy-opensearch-development] - - task: create-tenants - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_development_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_development_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: upload_tenant - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Create tenants for OpenSearch in development FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: upload-dashboards-objects-development - serial_groups: [bosh-development] - plan: - - in_parallel: - - get: pipeline-tasks - - get: opensearch-release - - get: opensearch-stemcell-jammy - - get: deploy-logs-opensearch-config - passed: [tenant-development] - trigger: true - - get: opensearch-development-deployment - passed: [tenant-development] - trigger: true - - task: upload-dashboards-objects - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_development_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_development_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: upload-dashboards-objects - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: FAILED to run upload-dashboards-objects for logs-opensearch in development - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: deploy-opensearch-staging - serial_groups: [bosh-staging] - plan: - - in_parallel: - - get: pipeline-tasks - - get: deploy-logs-opensearch-config - trigger: true - passed: [smoke-tests-development] - - get: opensearch-release - trigger: true - passed: [smoke-tests-development] - - get: opensearch-stemcell-jammy - trigger: true - passed: [smoke-tests-development] - - get: general-task - - get: terraform-yaml - resource: terraform-yaml-staging - trigger: true - - task: opensearch-manifest - image: general-task - config: - platform: linux - inputs: - - name: deploy-logs-opensearch-config - run: - path: sh - args: - - -exc - - | - bosh int deploy-logs-opensearch-config/opensearch-base.yml \ - -o deploy-logs-opensearch-config/opensearch-scaling-staging.yml \ - > opensearch-manifest/manifest.yml - outputs: - - name: opensearch-manifest - - task: terraform-secrets - image: general-task - file: deploy-logs-opensearch-config/ci/terraform-secrets.yml - - put: opensearch-staging-deployment - params: - manifest: opensearch-manifest/manifest.yml - releases: - - opensearch-release/*.tgz - stemcells: - - opensearch-stemcell-jammy/*.tgz - ops_files: - - deploy-logs-opensearch-config/opsfiles/enable-node-tls.yml - - deploy-logs-opensearch-config/opsfiles/enable-dashboard-dns.yml - - deploy-logs-opensearch-config/opsfiles/enable-dashboards-tls.yml - - deploy-logs-opensearch-config/opsfiles/enable-proxy-auth.yml - - deploy-logs-opensearch-config/opsfiles/enable-auth-proxy-route-staging.yml - - deploy-logs-opensearch-config/opsfiles/enable-syslog.yml - - deploy-logs-opensearch-config/opsfiles/cf-staging.yml - - deploy-logs-opensearch-config/opsfiles/opensearch-notification.yml - - deploy-logs-opensearch-config/opsfiles/add-keystore-credentials-errand.yml - vars_files: - - terraform-secrets/terraform.yml - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: FAILED to deploy logs-OpenSearch in staging - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: smoke-tests-staging - serial_groups: [bosh-staging] - plan: - - in_parallel: - - get: tests-timer - trigger: true - - get: pipeline-tasks - - get: opensearch-release - trigger: true - passed: [deploy-opensearch-staging] - - get: opensearch-stemcell-jammy - trigger: true - passed: [deploy-opensearch-staging] - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-staging] - trigger: true - - get: opensearch-staging-deployment - passed: [deploy-opensearch-staging] - trigger: true - - task: smoke-tests - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_staging_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_staging_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: smoke_tests - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Smoke tests for OpenSearch in staging FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: e2e-tests-staging - serial_groups: [bosh-staging] - plan: - - in_parallel: - - get: pipeline-tasks - - get: opensearch-release - trigger: true - passed: [deploy-opensearch-staging] - - get: opensearch-stemcell-jammy - trigger: true - passed: [deploy-opensearch-staging] - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-staging] - trigger: true - - get: opensearch-staging-deployment - trigger: true - passed: [deploy-opensearch-staging] - - get: one-hour-timer - trigger: true - - get: playwright-python - - get: general-task - - task: provision-test-user-cf-access - image: general-task - config: - platform: linux - inputs: - - name: deploy-logs-opensearch-config - run: - path: deploy-logs-opensearch-config/ci/provision-test-user-cf-access.sh - params: - CF_API_URL: ((staging-cf-api-url)) - CF_USERNAME: ((staging-cf-username)) - CF_PASSWORD: ((staging-cf-password)) - TEST_USER_1_USERNAME: ((staging-test-user-1-username)) - TEST_USER_2_USERNAME: ((staging-test-user-2-username)) - TEST_USER_3_USERNAME: ((staging-test-user-3-username)) - TEST_USER_4_USERNAME: ((staging-test-user-4-username)) - CF_ORG_1_NAME: ((test-org-1)) - CF_ORG_2_NAME: ((test-org-2)) - CF_ORG_3_NAME: ((test-org-3)) - - task: e2e-tests - image: playwright-python - file: deploy-logs-opensearch-config/ci/e2e-tests.yml - params: - AUTH_PROXY_URL: ((staging-auth-proxy-url)) - UAA_BASE_URL: ((staging-uaa-base-url)) - - CF_ORG_1_NAME: ((test-org-1)) - CF_ORG_2_NAME: ((test-org-2)) - CF_ORG_3_NAME: ((test-org-3)) - - TEST_USER_1_USERNAME: ((staging-test-user-1-username)) - TEST_USER_1_PASSWORD: ((staging-test-user-1-password)) - TEST_USER_1_TOTP_SEED: ((staging-test-user-1-totp-seed)) - - TEST_USER_2_USERNAME: ((staging-test-user-2-username)) - TEST_USER_2_PASSWORD: ((staging-test-user-2-password)) - TEST_USER_2_TOTP_SEED: ((staging-test-user-2-totp-seed)) - - TEST_USER_3_USERNAME: ((staging-test-user-3-username)) - TEST_USER_3_PASSWORD: ((staging-test-user-3-password)) - TEST_USER_3_TOTP_SEED: ((staging-test-user-3-totp-seed)) - - TEST_USER_4_USERNAME: ((staging-test-user-4-username)) - TEST_USER_4_PASSWORD: ((staging-test-user-4-password)) - TEST_USER_4_TOTP_SEED: ((staging-test-user-4-totp-seed)) - - SMTP_SENDER_HOST: ((log-alerts-smtp-sender-host)) - SMTP_SENDER_PORT: ((log-alerts-smtp-sender-port)) - SMTP_SENDER_FROM: ((staging-log-alerts-smtp-sender-from)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: E2E tests for OpenSearch in staging FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: tenant-staging - serial_groups: [bosh-staging] - plan: - - in_parallel: - - get: general-task - - get: pipeline-tasks - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-staging] - trigger: true - - get: opensearch-staging-deployment - trigger: true - passed: [deploy-opensearch-staging] - - task: create-tenants - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_staging_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_staging_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: upload_tenant - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Create tenants for OpenSearch in staging FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + - build-opensearch-release + - deploy-opensearch-development - - name: upload-dashboards-objects-staging - serial_groups: [bosh-staging] +jobs: + - name: build-opensearch-release plan: - in_parallel: - - get: pipeline-tasks - - get: opensearch-release - - get: opensearch-stemcell-jammy - - get: deploy-logs-opensearch-config - passed: [tenant-staging] - trigger: true - - get: opensearch-staging-deployment - passed: [tenant-staging] + - get: release-git-repo + resource: opensearch-release-git-repo trigger: true - - task: upload-dashboards-objects - file: pipeline-tasks/bosh-logs-errand.yml + - get: pipeline-tasks + - get: final-builds-dir-tarball + resource: opensearch-final-builds-dir-tarball + - get: releases-dir-tarball + resource: opensearch-releases-dir-tarball + - get: general-task + - task: run-tests + image: general-task + config: + platform: linux + inputs: + - name: release-git-repo + run: + path: sh + args: + - -exc + - | + cd release-git-repo + bundle install + # run release template unit tests + bundle exec rspec + - task: finalize-release + file: pipeline-tasks/finalize-bosh-release.yml + tags: [iaas] params: - BOSH_ENVIRONMENT: ((bosh_staging_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_staging_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: upload-dashboards-objects - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) + AWS_DEFAULT_REGION: ((aws_default_region)) + AWS_ACCESS_KEY_ID: ((aws_access_key_id)) + AWS_SECRET_ACCESS_KEY: ((aws_secret_access_key)) + PRIVATE_YML_CONTENT: |- + --- + blobstore: + options: + region: ((s3-bosh-blobstore-info.region)) + bucket_name: ((s3-bosh-blobstore-info.bucket_name)) + credentials_source: ((s3-bosh-blobstore-info.credentials_source)) + server_side_encryption: ((s3-bosh-blobstore-info.server_side_encryption)) + - in_parallel: + - put: opensearch-release + tags: [iaas] + params: + file: finalized-release/opensearch-*.tgz + - put: opensearch-final-builds-dir-tarball + tags: [iaas] + params: + file: finalized-release/final-builds-dir-opensearch.tgz + - put: opensearch-releases-dir-tarball + tags: [iaas] + params: + file: finalized-release/releases-dir-opensearch.tgz on_failure: put: slack - params: - <<: *slack-failure-params + params: &slack-failure-params text: | - :x: FAILED to run upload-dashboards-objects for logs-opensearch in staging + :x: FAILED to build OpenSearch BOSH release <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: plan-opensearch-production - serial_groups: [bosh-production] + channel: ((slack-channel-customer-success)) + username: ((slack-username)) + icon_url: ((slack-icon-url)) + - name: deploy-opensearch-development + serial_groups: [bosh-development] plan: - in_parallel: - get: pipeline-tasks - - get: deploy-logs-opensearch-config + - get: deploy-logs-platform-config trigger: true - passed: [smoke-tests-staging] - get: opensearch-release trigger: true - passed: [smoke-tests-staging] - get: opensearch-stemcell-jammy trigger: true - passed: [smoke-tests-staging] + - get: general-task - get: terraform-yaml - resource: terraform-yaml-production + resource: terraform-yaml-development trigger: true - - get: general-task - task: opensearch-manifest image: general-task - config: &build-prod-manifest-config + config: platform: linux inputs: - - name: deploy-logs-opensearch-config - - name: terraform-yaml + - name: deploy-logs-platform-config run: path: sh args: - -exc - | - bosh int deploy-logs-opensearch-config/opensearch-base.yml \ - -o deploy-logs-opensearch-config/opensearch-scaling-production.yml \ - > opensearch-manifest/manifest.yml + bosh int deploy-logs-platform-config/opensearch-base.yml \ + -o deploy-logs-platform-config/opensearch-scaling-development.yml \ + > opensearch-manifest/manifest.yml outputs: - name: opensearch-manifest - task: terraform-secrets image: general-task - file: deploy-logs-opensearch-config/ci/terraform-secrets.yml - - put: opensearch-production-deployment - params: &prod-deploy-params - dry_run: true + file: deploy-logs-platform-config/ci/terraform-secrets.yml + - put: opensearch-development-deployment + params: manifest: opensearch-manifest/manifest.yml + dry_run: true releases: - opensearch-release/*.tgz stemcells: - opensearch-stemcell-jammy/*.tgz ops_files: - - deploy-logs-opensearch-config/opsfiles/enable-node-tls.yml - - deploy-logs-opensearch-config/opsfiles/enable-dashboard-dns.yml - - deploy-logs-opensearch-config/opsfiles/enable-dashboards-tls.yml - - deploy-logs-opensearch-config/opsfiles/enable-proxy-auth.yml - - deploy-logs-opensearch-config/opsfiles/enable-auth-proxy-route-production.yml - - deploy-logs-opensearch-config/opsfiles/enable-syslog.yml - - deploy-logs-opensearch-config/opsfiles/cf-production.yml - - deploy-logs-opensearch-config/opsfiles/opensearch-notification.yml - - deploy-logs-opensearch-config/opsfiles/add-keystore-credentials-errand.yml + - deploy-logs-platform-config/opsfiles/enable-node-tls.yml + - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml + - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml + - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml + - deploy-logs-platform-config/opsfiles/enable-syslog.yml + - deploy-logs-platform-config/opsfiles/cf-development.yml + - deploy-logs-platform-config/opsfiles/opensearch-notification.yml + - deploy-logs-platform-config/opsfiles/add-keystore-credentials-errand.yml vars_files: - terraform-secrets/terraform.yml on_failure: @@ -1029,309 +129,16 @@ jobs: params: <<: *slack-failure-params text: | - :x: FAILED to plan deployment of logs-OpenSearch in production - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: deploy-opensearch-production - serial_groups: [bosh-production] - plan: - - in_parallel: - - get: pipeline-tasks - - get: deploy-logs-opensearch-config - passed: [plan-opensearch-production] - - get: opensearch-release - passed: [plan-opensearch-production] - - get: opensearch-stemcell-jammy - passed: [plan-opensearch-production] - - get: terraform-yaml - resource: terraform-yaml-production - - get: general-task - - task: opensearch-manifest - image: general-task - config: - <<: *build-prod-manifest-config - - task: terraform-secrets - image: general-task - file: deploy-logs-opensearch-config/ci/terraform-secrets.yml - - put: opensearch-production-deployment - params: - <<: *prod-deploy-params - dry_run: false - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: FAILED to deploy logs-OpenSearch in production - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: smoke-tests-production - serial_groups: [bosh-production] - plan: - - in_parallel: - - get: tests-timer - trigger: true - - get: pipeline-tasks - - get: opensearch-release - trigger: true - passed: [deploy-opensearch-production] - - get: opensearch-stemcell-jammy - trigger: true - passed: [deploy-opensearch-production] - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-production] - trigger: true - - get: opensearch-production-deployment - trigger: true - passed: [deploy-opensearch-production] - - task: smoke-tests - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: smoke_tests - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Smoke tests for OpenSearch in production FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: e2e-tests-production - serial_groups: [bosh-production] - plan: - - in_parallel: - - get: pipeline-tasks - - get: opensearch-release - trigger: true - passed: [deploy-opensearch-production] - - get: opensearch-stemcell-jammy - trigger: true - passed: [deploy-opensearch-production] - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-production] - trigger: true - - get: opensearch-production-deployment - trigger: true - passed: [deploy-opensearch-production] - - get: one-hour-timer - trigger: true - - get: playwright-python - - get: general-task - - task: provision-test-user-cf-access - image: general-task - config: - platform: linux - inputs: - - name: deploy-logs-opensearch-config - run: - path: deploy-logs-opensearch-config/ci/provision-test-user-cf-access.sh - params: - CF_API_URL: ((prod-cf-api-url)) - CF_USERNAME: ((production-cf-username)) - CF_PASSWORD: ((production-cf-password)) - TEST_USER_1_USERNAME: ((production-test-user-1-username)) - TEST_USER_2_USERNAME: ((production-test-user-2-username)) - TEST_USER_3_USERNAME: ((production-test-user-3-username)) - TEST_USER_4_USERNAME: ((production-test-user-4-username)) - CF_ORG_1_NAME: ((test-org-1)) - CF_ORG_2_NAME: ((test-org-2)) - CF_ORG_3_NAME: ((test-org-3)) - - task: e2e-tests - image: playwright-python - file: deploy-logs-opensearch-config/ci/e2e-tests.yml - params: - AUTH_PROXY_URL: ((production-auth-proxy-url)) - UAA_BASE_URL: ((production-uaa-base-url)) - - CF_ORG_1_NAME: ((test-org-1)) - CF_ORG_2_NAME: ((test-org-2)) - CF_ORG_3_NAME: ((test-org-3)) - - TEST_USER_1_USERNAME: ((production-test-user-1-username)) - TEST_USER_1_PASSWORD: ((production-test-user-1-password)) - TEST_USER_1_TOTP_SEED: ((production-test-user-1-totp-seed)) - - TEST_USER_2_USERNAME: ((production-test-user-2-username)) - TEST_USER_2_PASSWORD: ((production-test-user-2-password)) - TEST_USER_2_TOTP_SEED: ((production-test-user-2-totp-seed)) - - TEST_USER_3_USERNAME: ((production-test-user-3-username)) - TEST_USER_3_PASSWORD: ((production-test-user-3-password)) - TEST_USER_3_TOTP_SEED: ((production-test-user-3-totp-seed)) - - TEST_USER_4_USERNAME: ((production-test-user-4-username)) - TEST_USER_4_PASSWORD: ((production-test-user-4-password)) - TEST_USER_4_TOTP_SEED: ((production-test-user-4-totp-seed)) - - SMTP_SENDER_HOST: ((log-alerts-smtp-sender-host)) - SMTP_SENDER_PORT: ((log-alerts-smtp-sender-port)) - SMTP_SENDER_FROM: ((production-log-alerts-smtp-sender-from)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: E2E tests for OpenSearch in production FAILED - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - - name: tenant-production - serial_groups: [bosh-production] - plan: - - in_parallel: - - get: general-task - - get: pipeline-tasks - - get: deploy-logs-opensearch-config - passed: [deploy-opensearch-production] - trigger: true - - get: opensearch-production-deployment - trigger: true - passed: [deploy-opensearch-production] - - task: create-tenants - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: upload_tenant - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: Create tenants for OpenSearch in production FAILED + :x: FAILED to deploy logs-platform in development <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - - name: upload-dashboards-objects-production - serial_groups: [bosh-production] - plan: - - in_parallel: - - get: pipeline-tasks - - get: opensearch-release - - get: opensearch-stemcell-jammy - - get: deploy-logs-opensearch-config - passed: [tenant-production] - trigger: true - - get: opensearch-production-deployment - passed: [tenant-production] - trigger: true - - get: general-task - - task: upload-dashboards-objects - file: pipeline-tasks/bosh-logs-errand.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_DEPLOYMENT: logs-opensearch - BOSH_ERRAND: upload-dashboards-objects - BOSH_FLAGS: "--keep-alive" - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - # Updating the dashboards touches files on the os, reset the aide db - - task: reset-aide-opensearch_dashboards - image: general-task - file: deploy-logs-opensearch-config/ci/reset-aide.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - BOSH_DEPLOYMENT_NAME: logs-opensearch - BOSH_INSTANCE_NAME: opensearch_dashboards - - - name: reset-aide-opensearch-production - serial: true - serial_groups: [reset-aide] - plan: - - in_parallel: - - get: general-task - - get: deploy-logs-opensearch-config - - in_parallel: - - task: reset-aide-opensearch_dashboards - image: general-task - file: deploy-logs-opensearch-config/ci/reset-aide.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - BOSH_DEPLOYMENT_NAME: logs-opensearch - BOSH_INSTANCE_NAME: opensearch_dashboards - - task: reset-aide-ingestor_cloudwatch_logs - image: general-task - file: deploy-logs-opensearch-config/ci/reset-aide.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - BOSH_DEPLOYMENT_NAME: logs-opensearch - BOSH_INSTANCE_NAME: ingestor_cloudwatch_logs - - task: reset-aide-ingestor_s3 - image: general-task - file: deploy-logs-opensearch-config/ci/reset-aide.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - BOSH_DEPLOYMENT_NAME: logs-opensearch - BOSH_INSTANCE_NAME: ingestor_s3 - - task: reset-aide-ingestor_aws_metrics - image: general-task - file: deploy-logs-opensearch-config/ci/reset-aide.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - BOSH_DEPLOYMENT_NAME: logs-opensearch - BOSH_INSTANCE_NAME: ingestor_aws_metrics - - task: reset-aide-ingestor_s3 - image: general-task - file: deploy-logs-opensearch-config/ci/reset-aide.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - BOSH_DEPLOYMENT_NAME: logs-opensearch - BOSH_INSTANCE_NAME: ingestor_s3 - - task: reset-aide-maintenance - image: general-task - file: deploy-logs-opensearch-config/ci/reset-aide.yml - params: - BOSH_ENVIRONMENT: ((bosh_production_environment)) - BOSH_CLIENT: ((bosh_client)) - BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) - BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) - BOSH_DEPLOYMENT_NAME: logs-opensearch - BOSH_INSTANCE_NAME: maintenance - - resources: - name: opensearch-release-git-repo type: git source: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/opensearch-boshrelease - branch: main - - - name: opensearch-test-release-git-repo - type: git - source: - commit_verification_keys: ((cloud-gov-pgp-keys)) - uri: https://github.com/cloud-gov/opensearch-boshrelease - branch: ((test-opensearch-release-branch)) - - + branch: platform - name: opensearch-final-builds-dir-tarball type: s3-iam source: &bosh-releases-bucket-info @@ -1339,13 +146,11 @@ resources: region_name: ((aws-region)) server_side_encryption: AES256 versioned_file: final-builds-dir-opensearch.tgz - - name: opensearch-releases-dir-tarball type: s3-iam source: <<: *bosh-releases-bucket-info versioned_file: releases-dir-opensearch.tgz - - name: opensearch-release type: s3-iam source: @@ -1353,91 +158,32 @@ resources: region_name: ((aws-region)) regexp: opensearch-([\d\.]*).tgz server_side_encryption: AES256 - - - name: opensearch-test-release - type: s3-iam - source: - bucket: ((s3-bosh-releases-bucket)) - region_name: ((aws-region)) - regexp: opensearch-test-([\d\.]*).tgz - server_side_encryption: AES256 - - name: terraform-yaml-development type: s3-iam source: bucket: ((tf-state-bucket-development)) versioned_file: ((tf-state-file-development)) region_name: ((aws-region)) - - - name: terraform-yaml-staging - type: s3-iam - source: - bucket: ((tf-state-bucket-staging)) - versioned_file: ((tf-state-file-staging)) - region_name: ((aws-region)) - - - name: terraform-yaml-production - type: s3-iam - source: - bucket: ((tf-state-bucket-production)) - versioned_file: ((tf-state-file-production)) - region_name: ((aws-region)) - - - name: deploy-logs-opensearch-config + - name: deploy-logs-platform-config type: git source: commit_verification_keys: ((cloud-gov-pgp-keys)) - uri: https://github.com/cloud-gov/deploy-logs-opensearch.git + uri: https://github.com/cloud-gov/deploy-logs-platform.git branch: main - - - name: deploy-logs-opensearch-test-config - type: git - source: - commit_verification_keys: ((cloud-gov-pgp-keys)) - uri: https://github.com/cloud-gov/deploy-logs-opensearch.git - branch: ((deploy-logs-opensearch-config-test-branch)) - - name: opensearch-stemcell-jammy source: name: bosh-aws-xen-hvm-ubuntu-jammy-go_agent type: bosh-io-stemcell - - name: pipeline-tasks type: git source: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/cg-pipeline-tasks.git branch: main - - - name: tests-timer - type: time - source: - interval: 30m - - - name: one-hour-timer - type: time - source: - interval: 1h - - - name: ten-minute-timer - type: time - source: - interval: 10m - - name: slack type: slack-notification source: url: ((slack-webhook-url)) - - - name: opensearch-test-deployment - type: bosh-deployment - source: &bosh-params-development - target: ((bosh-director-info.development.environment)) - client: ((bosh-director-info.development.client)) - client_secret: ((bosh-director-info.development.client_secret)) - ca_cert: ((bosh-director-info.development.ca_cert)) - deployment: logs-opensearch-test - - name: opensearch-development-deployment type: bosh-deployment source: &bosh-params-development @@ -1445,26 +191,7 @@ resources: client: ((bosh-director-info.development.client)) client_secret: ((bosh-director-info.development.client_secret)) ca_cert: ((bosh-director-info.development.ca_cert)) - deployment: logs-opensearch - - - name: opensearch-staging-deployment - type: bosh-deployment - source: &bosh-params-staging - target: ((bosh-director-info.staging.environment)) - client: ((bosh-director-info.staging.client)) - client_secret: ((bosh-director-info.staging.client_secret)) - ca_cert: ((bosh-director-info.staging.ca_cert)) - deployment: logs-opensearch - - - name: opensearch-production-deployment - type: bosh-deployment - source: &bosh-params-production - target: ((bosh-director-info.production.environment)) - client: ((bosh-director-info.production.client)) - client_secret: ((bosh-director-info.production.client-secret)) - ca_cert: ((bosh-director-info.production.ca_cert)) - deployment: logs-opensearch - + deployment: logs-platform - name: general-task type: registry-image source: @@ -1474,24 +201,6 @@ resources: aws_region: us-gov-west-1 tag: latest - - name: playwright-python - type: registry-image - source: - aws_access_key_id: ((ecr_aws_key)) - aws_secret_access_key: ((ecr_aws_secret)) - repository: playwright-python - aws_region: us-gov-west-1 - tag: latest - - - name: weekly - type: time - source: - start: 12:00 AM - stop: 1:00 AM - location: America/New_York - days: [Thursday] - initial_version: true - resource_types: - name: registry-image type: registry-image @@ -1501,7 +210,6 @@ resource_types: repository: registry-image-resource aws_region: us-gov-west-1 tag: latest - - name: slack-notification type: registry-image source: @@ -1510,7 +218,6 @@ resource_types: repository: slack-notification-resource aws_region: us-gov-west-1 tag: latest - - name: bosh-deployment type: registry-image source: @@ -1519,7 +226,6 @@ resource_types: repository: bosh-deployment-resource aws_region: us-gov-west-1 tag: latest - - name: s3-iam type: registry-image source: @@ -1528,16 +234,6 @@ resource_types: repository: s3-resource aws_region: us-gov-west-1 tag: latest - - - name: time - type: registry-image - source: - aws_access_key_id: ((ecr_aws_key)) - aws_secret_access_key: ((ecr_aws_secret)) - repository: time-resource - aws_region: us-gov-west-1 - tag: latest - - name: git type: registry-image source: @@ -1545,4 +241,4 @@ resource_types: aws_secret_access_key: ((ecr_aws_secret)) repository: git-resource aws_region: us-gov-west-1 - tag: latest + tag: latest \ No newline at end of file diff --git a/ci/reset-aide.yml b/ci/reset-aide.yml index 0def41d..eb13a20 100644 --- a/ci/reset-aide.yml +++ b/ci/reset-aide.yml @@ -2,11 +2,11 @@ platform: linux inputs: - - { name: deploy-logs-opensearch-config } + - { name: deploy-logs-platform-config } run: - path: deploy-logs-opensearch-config/ci/reset-aide.sh + path: deploy-logs-platform-config/ci/reset-aide.sh params: BOSH_CA_CERT: diff --git a/ci/run-unit-tests.yml b/ci/run-unit-tests.yml index 997f563..625663f 100644 --- a/ci/run-unit-tests.yml +++ b/ci/run-unit-tests.yml @@ -1,13 +1,13 @@ platform: linux inputs: -- name: deploy-logs-opensearch-config +- name: deploy-logs-platform-config run: path: sh args: - -c - | - cd deploy-logs-opensearch-config/ + cd deploy-logs-platform-config/ python -m pip install -r requirements-test.txt python -m pytest tests diff --git a/ci/terraform-secrets.sh b/ci/terraform-secrets.sh index fdde38d..1f9cf18 100755 --- a/ci/terraform-secrets.sh +++ b/ci/terraform-secrets.sh @@ -1,6 +1,6 @@ #!/bin/bash bosh interpolate \ - deploy-logs-opensearch-config/varsfiles/terraform.yml \ + deploy-logs-platform-config/varsfiles/terraform.yml \ -l terraform-yaml/state.yml \ > terraform-secrets/terraform.yml diff --git a/ci/terraform-secrets.yml b/ci/terraform-secrets.yml index 469b72d..e2c7565 100644 --- a/ci/terraform-secrets.yml +++ b/ci/terraform-secrets.yml @@ -2,10 +2,10 @@ platform: linux inputs: -- name: deploy-logs-opensearch-config +- name: deploy-logs-platform-config - name: terraform-yaml outputs: - name: terraform-secrets run: - path: deploy-logs-opensearch-config/ci/terraform-secrets.sh + path: deploy-logs-platform-config/ci/terraform-secrets.sh diff --git a/ci/update-test-user-passwords.sh b/ci/update-test-user-passwords.sh index 7aacd08..ca48af5 100755 --- a/ci/update-test-user-passwords.sh +++ b/ci/update-test-user-passwords.sh @@ -14,7 +14,7 @@ for credential_name in $TEST_USER_CREDENTIAL_NAMES; do printf "updating password credential for %s\n\n" "$credential_name" # Generate a new password for the credential - PASSWORD_CREDENTIAL="/concourse/main/deploy-logs-opensearch/$credential_name" + PASSWORD_CREDENTIAL="/concourse/main/deploy-logs-platform/$credential_name" if ! credhub get -n "$PASSWORD_CREDENTIAL" > /dev/null; then credhub generate -n "$PASSWORD_CREDENTIAL" --type password else diff --git a/ci/upload-audit-events-s3.yml b/ci/upload-audit-events-s3.yml index 27f5ecf..368da5c 100644 --- a/ci/upload-audit-events-s3.yml +++ b/ci/upload-audit-events-s3.yml @@ -1,14 +1,14 @@ platform: linux inputs: -- name: deploy-logs-opensearch-config +- name: deploy-logs-platform-config run: path: sh args: - -c - | - cd deploy-logs-opensearch-config/ + cd deploy-logs-platform-config/ python -m pip install -r requirements-test.txt python ci/upload_audit_events_s3.py params: diff --git a/opensearch-base.yml b/opensearch-base.yml index d094a44..f414205 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -10,7 +10,7 @@ update: max_errors: 1 variables: [] -name: logs-opensearch +name: logs-platform releases: - {name: opensearch, version: latest} @@ -68,7 +68,7 @@ instance_groups: azs: [z1,z2] vm_type: t3.large\ vm_extensions: - - logs-opensearch-profile + - logs-platform-profile networks: - name: services update: @@ -110,7 +110,7 @@ instance_groups: stemcell: default azs: [z1,z2] vm_extensions: - - logs-opensearch-profile + - logs-platform-profile vm_type: t3.large networks: - name: services @@ -130,7 +130,7 @@ instance_groups: - name: opensearch_manager instances: 3 vm_extensions: - - logs-opensearch-profile + - logs-platform-profile - 15GB_ephemeral_disk jobs: - name: bpm @@ -155,8 +155,8 @@ instance_groups: properties: snort: rules: - - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"POST"; http_method; content: "logs-opensearch-app"; http_uri; content:"/_update"; http_uri; classtype:web-application-attack; sid:343080002; rev:1;)' - - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"DELETE"; http_method; content: "logs-opensearch-app"; http_uri; classtype:web-application-attack; sid:343080004; rev:1;)' + - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"POST"; http_method; content: "logs-platform-app"; http_uri; content:"/_update"; http_uri; classtype:web-application-attack; sid:343080002; rev:1;)' + - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"DELETE"; http_method; content: "logs-platform-app"; http_uri; classtype:web-application-attack; sid:343080004; rev:1;)' release: jammy-snort azs: [z1,z2] persistent_disk_type: logs_opensearch_os_master @@ -175,7 +175,7 @@ instance_groups: instances: 1 vm_extensions: - 20GB_ephemeral_disk - - logs-opensearch-ingestor-profile + - logs-platform-ingestor-profile jobs: - name: bpm release: bpm @@ -312,61 +312,6 @@ instance_groups: update: serial: true # Block on this job to create deploy group 3 -######################################################### -#4th deploy group - archiver -######################################################### -- name: archiver - instances: 1 - jobs: - - name: bpm - release: bpm - - name: opensearch - consumes: *consumes-opensearch-manager - properties: - opensearch: - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - migrate_data_path: true - release: opensearch - - consumes: - opensearch: - from: opensearch_manager - ip_addresses: true - name: archiver_syslog - properties: - logstash: - queue: - max_bytes: 30gb - logstash_archiver: - files: 16384 - logstash_ingestor: - outputs: - - options: - bucket: (( grab terraform_outputs.logs_opensearch_archive_bucket_name )) - encoding: gzip - prefix: '%{+yyyy/MM/dd/HH/mm}' - region: (( grab terraform_outputs.vpc_region )) - server_side_encryption: true - time_file: 5 - validate_credentials_on_root_bucket: false - plugin: s3 - syslog_tls: - port: 7891 - ssl_cert: ((archiver_syslog_server_tls.certificate)) - ssl_key: ((archiver_syslog_server_tls.private_key)) - release: opensearch - azs: [z1,z2] - networks: - - name: services - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - vm_extensions: - - logs-opensearch-ingestor-profile - - 20GB_ephemeral_disk - vm_type: t3.medium - update: - serial: true # Block on this job to create deploy group 4 - ######################################################### #5th deploy group ingestors # Ingestors are run second to last and updated one at a time to avoid collisions @@ -388,6 +333,22 @@ instance_groups: - name: ingestor_syslog consumes: *consumes-opensearch-manager properties: + outputs: + - plugin: s3 + options: + region: ((region)) + bucket: ((bucket)) + access_key_id: ((access_key_id)) + secret_access_key: ((secret_access_key)) + server_side_encryption: true + prefix: "%{+yyyy/MM/dd/HH/mm}" + encoding: "gzip" + temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp + # note that this is different than in the archivers + # here, we're uploading the parsed message as json, not the raw log line + codec: json + - plugin: opensearch + options: {} logstash: jvm_options: - -Dlog4j2.formatMsgNoLookups=true @@ -425,250 +386,13 @@ instance_groups: azs: [z1,z2] vm_type: t3.medium vm_extensions: - - logs-opensearch-ingestor-profile + - logs-platform-ingestor-profile - 20GB_ephemeral_disk networks: - name: services update: serial: true # Block on this job to create deploy group 5 -- name: ingestor_s3 - instances: 1 - jobs: - - name: bpm - release: bpm - - name: opensearch - release: opensearch - consumes: *consumes-opensearch-manager - properties: - opensearch: - heap_size: 1G - http_host: 127.0.0.1 - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - - consumes: *consumes-opensearch-manager - name: ingestor_s3 - properties: - logstash: - type: audit - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - queue: - max_bytes: 30gb - logstash_ingestor: - s3: - bucket: ((audit_bucket)) - region: ((region)) - syslog_tls: - port: 6972 - ssl_cert: ((ingestor_syslog_server_tls.certificate)) - ssl_key: ((ingestor_syslog_server_tls.private_key)) - logstash_parser: - opensearch: - data_hosts: - - localhost - index: ((alias)) - index_type: '%{@type}' - ssl: - ca: ((opensearch_node.ca)) - certificate: ((logstash.certificate)) - private_key: ((logstash.private_key)) - provides: - ingestor: - as: ingestor_s3 - release: opensearch - - name: deployment_lookup_config - release: opensearch - azs: [z1,z2] - networks: - - name: services - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - vm_extensions: - - logs-opensearch-ingestor-profile - - 20GB_ephemeral_disk - vm_type: t3.large - update: - serial: true # Block on this job to create deploy group 6 - -- name: ingestor_aws_metrics - instances: 1 - jobs: - - name: bpm - release: bpm - - name: opensearch - release: opensearch - consumes: *consumes-opensearch-manager - properties: - opensearch: - heap_size: 1G - http_host: 127.0.0.1 - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - - consumes: *consumes-opensearch-manager - name: ingestor_s3 - properties: - logstash: - type: metric - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - queue: - max_bytes: 30gb - logstash_ingestor: - s3: - bucket: ((metric_stream_bucket)) - region: ((region)) - syslog_tls: - port: 6972 - ssl_cert: ((ingestor_syslog_server_tls.certificate)) - ssl_key: ((ingestor_syslog_server_tls.private_key)) - logstash_parser: - opensearch: - data_hosts: - - localhost - index: ((metric_alias)) - index_type: '%{@type}' - ssl: - ca: ((opensearch_node.ca)) - certificate: ((logstash.certificate)) - private_key: ((logstash.private_key)) - provides: - ingestor: - as: ingestor_s3 - release: opensearch - - name: deployment_lookup_config - release: opensearch - azs: [z1,z2] - networks: - - name: services - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - vm_extensions: - - logs-opensearch-ingestor-profile - - 20GB_ephemeral_disk - vm_type: t3.large - update: - serial: true # Block on this job to create deploy group 6 - -- name: ingestor_cloudwatch_logs - instances: 1 - jobs: - - name: bpm - release: bpm - - name: opensearch - release: opensearch - consumes: *consumes-opensearch-manager - properties: - opensearch: - heap_size: 1G - http_host: 127.0.0.1 - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - - consumes: *consumes-opensearch-manager - name: ingestor_s3 - properties: - logstash: - type: cloudwatch - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - queue: - max_bytes: 30gb - logstash_ingestor: - s3: - bucket: ((cloudwatch_bucket)) - region: ((region)) - syslog_tls: - port: 6972 - ssl_cert: ((ingestor_syslog_server_tls.certificate)) - ssl_key: ((ingestor_syslog_server_tls.private_key)) - logstash_parser: - opensearch: - data_hosts: - - localhost - index: ((alias)) - index_type: '%{@type}' - ssl: - ca: ((opensearch_node.ca)) - certificate: ((logstash.certificate)) - private_key: ((logstash.private_key)) - provides: - ingestor: - as: ingestor_s3 - release: opensearch - - name: deployment_lookup_config - release: opensearch - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - azs: [z1,z2] - vm_type: t3.large - vm_extensions: - - logs-opensearch-ingestor-profile - - 20GB_ephemeral_disk - networks: - - name: services - update: - serial: true # Block on this job to create deploy group 7 - -- name: ingestor_falco - instances: 0 - jobs: - - name: bpm - release: bpm - - name: opensearch - release: opensearch - consumes: *consumes-opensearch-manager - properties: - opensearch: - heap_size: 1G - http_host: 127.0.0.1 - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - - consumes: *consumes-opensearch-manager - name: ingestor_s3 - properties: - logstash: - type: falco - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - queue: - max_bytes: 30gb - logstash_ingestor: - s3: - bucket: ((falco_bucket)) - region: ((region)) - prefix: falco - syslog_tls: - port: 6972 - ssl_cert: ((ingestor_syslog_server_tls.certificate)) - ssl_key: ((ingestor_syslog_server_tls.private_key)) - logstash_parser: - opensearch: - data_hosts: - - localhost - index: ((alias)) - index_type: '%{@type}' - ssl: - ca: ((opensearch_node.ca)) - certificate: ((logstash.certificate)) - private_key: ((logstash.private_key)) - provides: - ingestor: - as: ingestor_s3 - release: opensearch - - name: deployment_lookup_config - release: opensearch - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - azs: [z1,z2] - vm_type: t3.large - vm_extensions: - - logs-opensearch-ingestor-profile - - 20GB_ephemeral_disk - networks: - - name: services - update: - serial: true # Block on this job to create deploy group 8 - ######################################################### # 9th deploy group - dashboards # Dashboard is ran last as it checks the types of other instances when restarting. all dashboard nodes must either be updated at the same @@ -676,7 +400,7 @@ instance_groups: # IF DASHBOARD IS UPDATING WITH A OLD VERSION BOTH WILL ERROR ######################################################### - name: opensearch_dashboards - instances: 2 + instances: 1 jobs: - name: bpm release: bpm @@ -709,7 +433,7 @@ instance_groups: release: opensearch vm_extensions: - 15GB_ephemeral_disk - - logs-opensearch-profile + - logs-platform-profile stemcell: default azs: [z1] networks: diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 226645d..458732c 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -1,15 +1,15 @@ - type: replace path: /instance_groups/name=opensearch_manager?/vm_type? - value: m6i.large + value: t3.large - type: replace path: /instance_groups/name=opensearch_manager?/instances? - value: 2 + value: 1 - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 3 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? @@ -41,7 +41,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards?/instances? - value: 2 + value: 1 - type: replace path: /instance_groups/name=opensearch_dashboards?/vm_type? @@ -55,15 +55,6 @@ serial: true - -- type: replace - path: /instance_groups/name=archiver?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=archiver?/vm_type? - value: m6i.large - - type: replace path: /instance_groups/name=ingestor?/instances? value: 1 @@ -72,21 +63,6 @@ path: /instance_groups/name=ingestor?/vm_type? value: m6i.large -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_s3?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_falco?/instances? - value: 1 - type: replace @@ -149,7 +125,7 @@ - type: replace path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/bucket? - value: ((logs-opensearch-cf-archiver-bucket)) + value: ((logs-platform-cf-archiver-bucket)) - type: replace path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/region? diff --git a/opensearch-scaling-production.yml b/opensearch-scaling-production.yml deleted file mode 100644 index 13984eb..0000000 --- a/opensearch-scaling-production.yml +++ /dev/null @@ -1,175 +0,0 @@ -- type: replace - path: /instance_groups/name=opensearch_manager?/vm_type? - value: m6i.xlarge - - -- type: replace - path: /instance_groups/name=opensearch_manager?/instances? - value: 3 - - - -- type: replace - path: /instance_groups/name=opensearch_data?/instances? - value: 24 - - -- type: replace - path: /instance_groups/name=opensearch_data?/vm_type? - value: r6i.2xlarge - -- type: replace - path: /instance_groups/name=opensearch_data?/update? - value: - max_in_flight: 1 - canaries: 1 - serial: true - -- type: replace - path: /instance_groups/name=opensearch_old_data?/instances? - value: 24 - - -- type: replace - path: /instance_groups/name=opensearch_old_data?/vm_type? - value: r6i.xlarge - -- type: replace - path: /instance_groups/name=opensearch_old_data?/update? - value: - max_in_flight: 1 - canaries: 1 - serial: true - -- type: replace - path: /instance_groups/name=opensearch_dashboards?/instances? - value: 2 - -- type: replace - path: /instance_groups/name=opensearch_dashboards?/vm_type? - value: t3.xlarge - -- type: replace - path: /instance_groups/name=opensearch_dashboards/update? - value: - max_in_flight: 100% - canaries: 100% - serial: true - - - -- type: replace - path: /instance_groups/name=archiver?/instances? - value: 3 - -- type: replace - path: /instance_groups/name=archiver?/vm_type? - value: r6i.xlarge - -- type: replace - path: /instance_groups/name=ingestor?/instances? - value: 12 - -- type: replace - path: /instance_groups/name=ingestor?/vm_type? - value: r6i.xlarge.logsearch.ingestor - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics?/vm_type? - value: r6i.xlarge - -- type: replace - path: /instance_groups/name=ingestor_s3?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? - value: - index_pattern: logs-app-* - long_time_interval: 10m - minimum: 1000 - run: true - time_field: '@timestamp' - time_interval: 5m - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/cloudwatch_count_test? - value: - minimum: 100 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/metric_count_test? - value: - minimum: 100 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/audit_count_test? - value: - minimum: 100 - - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_document_size? - value: 30gb - - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_index_age? - value: 1d - - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/shard_count? - value: 18 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/metrics_shard_count? - value: 6 - - - -- type: replace - path: /instance_groups/name=maintenance?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=maintenance?/vm_type? - value: t3.large - - - - -- type: replace - path: /addons? - value: - - name: bosh-dns-aliases - jobs: - - name: bosh-dns-aliases - release: bosh-dns-aliases - properties: - aliases: - - domain: nats.service.cf.internal - targets: - - deployment: cf-production - domain: bosh - instance_group: nats - network: default - query: '*' - - -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/bucket? - value: ((logs-opensearch-cf-archiver-bucket)) - -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/region? - value: us-gov-west-1 - diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml deleted file mode 100644 index 42ab567..0000000 --- a/opensearch-scaling-staging.yml +++ /dev/null @@ -1,173 +0,0 @@ -- type: replace - path: /instance_groups/name=opensearch_manager?/vm_type? - value: m6i.large - -- type: replace - path: /instance_groups/name=opensearch_manager?/instances? - value: 3 - - -- type: replace - path: /instance_groups/name=opensearch_data?/instances? - value: 3 - -- type: replace - path: /instance_groups/name=opensearch_data?/vm_type? - value: r6i.large - -- type: replace - path: /instance_groups/name=opensearch_data?/update? - value: - max_in_flight: 1 - canaries: 1 - serial: true - - -- type: replace - path: /instance_groups/name=opensearch_old_data?/instances? - value: 3 - -- type: replace - path: /instance_groups/name=opensearch_old_data?/vm_type? - value: r6i.large - -- type: replace - path: /instance_groups/name=opensearch_old_data?/update? - value: - max_in_flight: 1 - canaries: 1 - serial: true - - -- type: replace - path: /instance_groups/name=opensearch_dashboards?/instances? - value: 2 - -- type: replace - path: /instance_groups/name=opensearch_dashboards?/vm_type? - value: t3.xlarge - -- type: replace - path: /instance_groups/name=opensearch_dashboards/update? - value: - max_in_flight: 100% - canaries: 100% - serial: true - - - -- type: replace - path: /instance_groups/name=archiver?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=archiver?/vm_type? - value: m6i.large - -- type: replace - path: /instance_groups/name=ingestor?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor?/vm_type? - value: m6i.large - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_falco?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_s3?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? - value: - index_pattern: logs-app-* - long_time_interval: 20m - minimum: 150 - run: true - time_field: '@timestamp' - time_interval: 5m - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/cloudwatch_count_test? - value: - minimum: 2 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/metric_count_test? - value: - minimum: 100 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/audit_count_test? - value: - minimum: 10 - - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_document_size? - value: 3gb - - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_index_age? - value: 1d - - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/shard_count? - value: 2 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/metrics_shard_count? - value: 2 - - - -- type: replace - path: /instance_groups/name=maintenance?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=maintenance?/vm_type? - value: t3.large - - - - -- type: replace - path: /addons? - value: - - name: bosh-dns-aliases - jobs: - - name: bosh-dns-aliases - release: bosh-dns-aliases - properties: - aliases: - - domain: nats.service.cf.internal - targets: - - deployment: cf-staging - domain: bosh - instance_group: nats - network: default - query: '*' - - -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/bucket? - value: ((logs-opensearch-cf-archiver-bucket)) - -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/region? - value: us-gov-west-1 - diff --git a/opensearch-scaling-testing.yml b/opensearch-scaling-testing.yml deleted file mode 100644 index fac948e..0000000 --- a/opensearch-scaling-testing.yml +++ /dev/null @@ -1,158 +0,0 @@ -- type: replace - path: /instance_groups/name=opensearch_manager?/vm_type? - value: t3.medium - -- type: replace - path: /instance_groups/name=opensearch_manager?/instances? - value: 1 - - -- type: replace - path: /instance_groups/name=opensearch_data?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=opensearch_data?/vm_type? - value: r6i.large - -- type: replace - path: /instance_groups/name=opensearch_data?/update? - value: - max_in_flight: 1 - canaries: 1 - serial: true -- type: replace - path: /instance_groups/name=opensearch_data?/jobs?/name=opensearch?/properties?/opensearch?/health?/disable_post_start? - value: true - - -- type: replace - path: /instance_groups/name=opensearch_old_data?/instances? - value: 0 - -- type: replace - path: /instance_groups/name=opensearch_old_data?/vm_type? - value: r6i.large - -- type: replace - path: /instance_groups/name=opensearch_old_data?/update? - value: - max_in_flight: 1 - canaries: 1 - serial: true - -- type: replace - path: /instance_groups/name=opensearch_old_data?/jobs?/name=opensearch?/properties?/opensearch?/health?/disable_post_start? - value: true - -- type: replace - path: /instance_groups/name=opensearch_dashboards?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=opensearch_dashboards?/vm_type? - value: t3.xlarge - -- type: replace - path: /instance_groups/name=opensearch_dashboards/update? - value: - max_in_flight: 100% - canaries: 100% - serial: true - - - -- type: replace - path: /instance_groups/name=archiver?/instances? - value: 0 - -- type: replace - path: /instance_groups/name=archiver?/vm_type? - value: m6i.large - -- type: replace - path: /instance_groups/name=ingestor?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor?/vm_type? - value: m6i.large - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs?/instances? - value: 0 - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_falco?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=ingestor_s3?/instances? - value: 0 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? - value: - index_pattern: logs-app-* - minimum: 1000 - run: false - time_field: '@timestamp' - time_interval: 5m - - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_document_size? - value: 3gb - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_index_age? - value: 1d - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/shard_count? - value: 1 - -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/metrics_shard_count? - value: 1 - - -- type: replace - path: /instance_groups/name=maintenance?/instances? - value: 1 - -- type: replace - path: /instance_groups/name=maintenance?/vm_type? - value: t3.large - - -- type: replace - path: /addons? - value: - - name: bosh-dns-aliases - jobs: - - name: bosh-dns-aliases - release: bosh-dns-aliases - properties: - aliases: - - domain: nats.service.cf.internal - targets: - - deployment: cf-development - domain: bosh - instance_group: nats - network: default - query: '*' - - -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/bucket? - value: ((logs-opensearch-cf-archiver-bucket)) - -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/region? - value: us-gov-west-1 - diff --git a/opsfiles/add-keystore-credentials-errand.yml b/opsfiles/add-keystore-credentials-errand.yml index 702662f..e4c6d57 100644 --- a/opsfiles/add-keystore-credentials-errand.yml +++ b/opsfiles/add-keystore-credentials-errand.yml @@ -68,43 +68,3 @@ path: /instance_groups/name=ingestor/jobs/name=opensearch/properties/opensearch?/notifications? value: <<: *add-credential-map - -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/- - value: - <<: *add-keystore-credentials-errand - -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch?/notifications? - value: - <<: *add-credential-map - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/- - value: - <<: *add-keystore-credentials-errand - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/name=opensearch/properties/opensearch?/notifications? - value: - <<: *add-credential-map - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/- - value: - <<: *add-keystore-credentials-errand - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/name=opensearch/properties/opensearch?/notifications? - value: - <<: *add-credential-map - -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/- - value: - <<: *add-keystore-credentials-errand - -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/name=opensearch/properties/opensearch?/notifications? - value: - <<: *add-credential-map \ No newline at end of file diff --git a/opsfiles/enable-auth-proxy-route-dev.yml b/opsfiles/enable-auth-proxy-route-dev.yml index cd2918b..c9e3ea5 100644 --- a/opsfiles/enable-auth-proxy-route-dev.yml +++ b/opsfiles/enable-auth-proxy-route-dev.yml @@ -9,7 +9,7 @@ # add address for auth proxy route - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/route_registrar?/routes?/name=opensearch-auth-proxy/uris?/- - value: logs.dev.us-gov-west-1.aws-us-gov.cloud.gov + value: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/consumes?/nats-tls?/deployment? diff --git a/opsfiles/enable-dashboard-dns.yml b/opsfiles/enable-dashboard-dns.yml index b565906..2b615bc 100644 --- a/opsfiles/enable-dashboard-dns.yml +++ b/opsfiles/enable-dashboard-dns.yml @@ -5,6 +5,6 @@ targets: - query: '*' instance_group: opensearch_dashboards - deployment: logs-opensearch + deployment: logs-platform network: services domain: bosh \ No newline at end of file diff --git a/opsfiles/enable-node-tls.yml b/opsfiles/enable-node-tls.yml index eb1bb6b..855da4f 100644 --- a/opsfiles/enable-node-tls.yml +++ b/opsfiles/enable-node-tls.yml @@ -106,23 +106,6 @@ certificate: ((opensearch_dashboard.certificate)) private_key: ((opensearch_dashboard.private_key)) -# archiver -- type: replace - path: /instance_groups/name=archiver/jobs/name=opensearch/properties?/opensearch?/http_host? - value: 127.0.0.1 - -- type: replace - path: /instance_groups/name=archiver/jobs/name=opensearch/properties/opensearch?/admin? - value: *admin-tls-properties - -- type: replace - path: /instance_groups/name=archiver/jobs/name=opensearch/properties/opensearch?/node?/ssl? - value: *node-tls-properties - -- type: replace - path: /instance_groups/name=archiver/jobs/name=opensearch/properties?/opensearch?/http?/ssl? - value: *http-tls-properties - # ingestor - type: replace path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/http_host? @@ -140,82 +123,6 @@ path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/http?/ssl? value: *http-tls-properties -# ingestor_cloudwatch_logs -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/name=opensearch/properties?/opensearch?/http_host? - value: 127.0.0.1 - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/name=opensearch/properties/opensearch?/admin? - value: *admin-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/name=opensearch/properties/opensearch?/node?/ssl? - value: *node-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/name=opensearch/properties?/opensearch?/http?/ssl? - value: *http-tls-properties - -# ingestor_s3 -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties?/opensearch?/http_host? - value: 127.0.0.1 - -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch?/admin? - value: *admin-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch?/node?/ssl? - value: *node-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties?/opensearch?/http?/ssl? - value: *http-tls-properties - -# ingestor_aws_metrics -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/name=opensearch/properties?/opensearch?/http_host? - value: 127.0.0.1 - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/name=opensearch/properties/opensearch?/admin? - value: *admin-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/name=opensearch/properties/opensearch?/node?/ssl? - value: *node-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/name=opensearch/properties?/opensearch?/http?/ssl? - value: *http-tls-properties - -# ingestor_falco -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/name=opensearch/properties?/opensearch?/http_host? - value: 127.0.0.1 - -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/name=opensearch/properties/opensearch?/admin? - value: *admin-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/name=opensearch/properties/opensearch?/node?/ssl? - value: *node-tls-properties - -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/name=opensearch/properties?/opensearch?/http?/ssl? - value: *http-tls-properties - -# smoke_tests -# - type: replace -# path: /instance_groups/name=maintenance/jobs/name=smoke_tests/properties?/smoke_tests?/opensearch?/ssl? -# value: -# ca: ((opensearch_node.ca)) -# certificate: ((smoketest.certificate)) -# private_key: ((smoketest.private_key)) - # variables - type: replace path: /variables/name=opensearch_ca? diff --git a/opsfiles/enable-proxy-auth.yml b/opsfiles/enable-proxy-auth.yml index 434aab9..8950a3f 100644 --- a/opsfiles/enable-proxy-auth.yml +++ b/opsfiles/enable-proxy-auth.yml @@ -39,25 +39,6 @@ path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth value: true -# ingestor_cloudwatch_logs -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth - value: true - -# ingestor_s3 -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth - value: true -# ingestor_aws_metrics -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth - value: true - -# ingestor_falco -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth - value: true - # add variable for auth proxy certs - type: replace path: /variables/name=auth_proxy? diff --git a/opsfiles/enable-syslog.yml b/opsfiles/enable-syslog.yml index cb64e75..0e2e0f6 100644 --- a/opsfiles/enable-syslog.yml +++ b/opsfiles/enable-syslog.yml @@ -6,9 +6,9 @@ name: ingestor_syslog_client_tls options: alternative_names: - - logs-opensearch.ingestor.service.cf.internal + - logs-platform.ingestor.service.cf.internal ca: opensearch_ca - common_name: logs-opensearch.ingestor.service.cf.internal + common_name: logs-platform.ingestor.service.cf.internal extended_key_usage: - client_auth type: certificate @@ -20,9 +20,9 @@ name: ingestor_syslog_server_tls options: alternative_names: - - logs-opensearch.ingestor.service.cf.internal + - logs-platform.ingestor.service.cf.internal ca: opensearch_ca - common_name: logs-opensearch.ingestor.service.cf.internal + common_name: logs-platform.ingestor.service.cf.internal extended_key_usage: - server_auth type: certificate @@ -34,9 +34,9 @@ name: archiver_syslog_client_tls options: alternative_names: - - logs-opensearch.archiver.service.cf.internal + - logs-platform.archiver.service.cf.internal ca: opensearch_ca - common_name: logs-opensearch.archiver.service.cf.internal + common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - client_auth type: certificate @@ -48,9 +48,9 @@ name: archiver_syslog_server_tls options: alternative_names: - - logs-opensearch.archiver.service.cf.internal + - logs-platform.archiver.service.cf.internal ca: opensearch_ca - common_name: logs-opensearch.archiver.service.cf.internal + common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - server_auth type: certificate @@ -62,9 +62,9 @@ name: archiver_syslog_client_tls options: alternative_names: - - logs-opensearch.archiver.service.cf.internal + - logs-platform.archiver.service.cf.internal ca: opensearch_ca - common_name: logs-opensearch.archiver.service.cf.internal + common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - client_auth type: certificate @@ -76,9 +76,9 @@ name: archiver_syslog_server_tls options: alternative_names: - - logs-opensearch.archiver.service.cf.internal + - logs-platform.archiver.service.cf.internal ca: opensearch_ca - common_name: logs-opensearch.archiver.service.cf.internal + common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - server_auth type: certificate diff --git a/opsfiles/opensearch-notification.yml b/opsfiles/opensearch-notification.yml index 762ac81..f4fb92a 100644 --- a/opsfiles/opensearch-notification.yml +++ b/opsfiles/opensearch-notification.yml @@ -40,42 +40,6 @@ keystorepass: ((keystore_password)) defaultkeystorepass: ((default_keystore_password)) - -- type: replace - path: /instance_groups/name=ingestor_cloudwatch_logs/jobs/name=opensearch/properties/opensearch/notification? - value: - keystorepass: ((keystore_password)) - defaultkeystorepass: ((default_keystore_password)) - - -- type: replace - path: /instance_groups/name=ingestor_aws_metrics/jobs/name=opensearch/properties/opensearch/notification? - value: - keystorepass: ((keystore_password)) - defaultkeystorepass: ((default_keystore_password)) - - -- type: replace - path: /instance_groups/name=ingestor_falco/jobs/name=opensearch/properties/opensearch/notification? - value: - ca: ((notification_ca.ca)) - certificate: ((notification_ca.certificate)) - from: ((notification_from)) - host: ((notification_host)) - keystorepass: ((keystore_password)) - defaultkeystorepass: ((default_keystore_password)) - password: ((notification_password)) - port: ((notification_port)) - privatekey: ((notification_ca.private_key)) - username: ((notification_username)) - -- type: replace - path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch/notification? - value: - keystorepass: ((keystore_password)) - defaultkeystorepass: ((default_keystore_password)) - - - type: replace path: /instance_groups/name=maintenance/jobs/name=opensearch/properties/opensearch/notification? value: diff --git a/opsfiles/test-name.yml b/opsfiles/test-name.yml index d865d5a..8907edc 100644 --- a/opsfiles/test-name.yml +++ b/opsfiles/test-name.yml @@ -1,3 +1,3 @@ - type: replace path: /name - value: logs-opensearch-test + value: logs-platform-test diff --git a/prod-opensearch-ingestors.yml b/prod-opensearch-ingestors.yml deleted file mode 100644 index 904f2a0..0000000 --- a/prod-opensearch-ingestors.yml +++ /dev/null @@ -1,74 +0,0 @@ -instance_groups: -- &opensearch-ingestor-config - name: ingestor_opensearch_s3 - instances: 1 - jobs: - - name: bpm - release: bpm - - name: opensearch - release: opensearch - consumes: - opensearch: - from: opensearch_manager - ip_addresses: true - properties: - opensearch: - heap_size: 1G - http_host: 127.0.0.1 - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - - name: ingestor_syslog - consumes: - opensearch: - from: opensearch_manager - ip_addresses: true - properties: - logstash: - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - queue: - max_bytes: 40gb - logstash_ingestor: - ingest_syslog_enabled: false - health: - disable_post_start: true - syslog_tls: - port: 6972 - ssl_cert: ((ingestor_syslog_server_tls.certificate)) - ssl_key: ((ingestor_syslog_server_tls.private_key)) - logstash_parser: - inputs: - - plugin: s3 - options: - bucket: (( grab terraform_outputs.logs_opensearch_archive_bucket_name )) - region: (( grab terraform_outputs.vpc_region )) - type: syslog - deployment_dictionary: - - /var/vcap/packages/base-logstash-filters/deployment_lookup.yml - - /var/vcap/jobs/deployment_lookup_config/config/deployment_lookup.yml - filters: - - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-filters-default.conf - opensearch: - data_hosts: - - localhost - index: "logs-app-%{+YYYY.MM.dd}" - index_type: '%{@type}' - ssl: - ca: ((opensearch_node.ca)) - certificate: ((logstash.certificate)) - private_key: ((logstash.private_key)) - provides: - ingestor: - as: ingestor_link - release: opensearch - - name: deployment_lookup_config - release: opensearch - vm_type: r6i.xlarge.logsearch.ingestor - networks: - - name: services - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - azs: - - z1 - vm_extensions: - - logs-opensearch-ingestor-profile diff --git a/scripts/download-e2e-ci-results.sh b/scripts/download-e2e-ci-results.sh index ce15efb..7f17ea3 100755 --- a/scripts/download-e2e-ci-results.sh +++ b/scripts/download-e2e-ci-results.sh @@ -9,8 +9,8 @@ fi ENVIRONMENT=${2:-production} -CI_TASK_TARGET="fly -t ${FLY_TARGET:=ci} intercept -j deploy-logs-opensearch/e2e-tests-$ENVIRONMENT -s e2e-tests -b $BUILD_NUMBER" -TEST_RESULTS_DIR="deploy-logs-opensearch-config/test-results" +CI_TASK_TARGET="fly -t ${FLY_TARGET:=ci} intercept -j deploy-logs-platform/e2e-tests-$ENVIRONMENT -s e2e-tests -b $BUILD_NUMBER" +TEST_RESULTS_DIR="deploy-logs-platform-config/test-results" LOCAL_TARGET_DIR="ci-test-results" for test_dir in $($CI_TASK_TARGET -- ls $TEST_RESULTS_DIR); do From 6b0e7111d153df75ceffe0967782976062695c83 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 11:36:32 -0400 Subject: [PATCH 002/165] fixing git resource --- ci/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 814320a..589ef5e 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -168,8 +168,8 @@ resources: type: git source: commit_verification_keys: ((cloud-gov-pgp-keys)) - uri: https://github.com/cloud-gov/deploy-logs-platform.git - branch: main + uri: https://github.com/cloud-gov/deploy-platform-opensearch.git + branch: platform - name: opensearch-stemcell-jammy source: name: bosh-aws-xen-hvm-ubuntu-jammy-go_agent From 1f98624a112c2b0f51242e543049a553c5b5aa56 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 11:41:37 -0400 Subject: [PATCH 003/165] getting past manifest stage --- opensearch-base.yml | 8 ++++---- opensearch-scaling-development.yml | 8 -------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index f414205..daa51f9 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -231,10 +231,10 @@ instance_groups: bucket: ((cloudwatch_bucket)) s3_metric: bucket: ((metric_stream_bucket)) - tls: - use_tls: true - certificate: ((archiver_syslog_client_tls.certificate)) - private_key: ((archiver_syslog_client_tls.private_key)) + # tls: + # use_tls: true + # certificate: ((archiver_syslog_client_tls.certificate)) + # private_key: ((archiver_syslog_client_tls.private_key)) release: opensearch - name: upload_opensearch_config consumes: *consumes-opensearch-manager diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 458732c..199ba67 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -123,11 +123,3 @@ query: '*' -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/bucket? - value: ((logs-platform-cf-archiver-bucket)) - -- type: replace - path: /instance_groups/name=archiver?/jobs?/name=archiver_syslog?/properties?/logstash_ingestor?/outputs?/0/options/region? - value: us-gov-west-1 - From cb6c8752409c0e428296c7bbd2846c2cd1ddd93b Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 13:34:02 -0400 Subject: [PATCH 004/165] seperation --- opsfiles/add-keystore-credentials-errand.yml | 10 ---------- opsfiles/opensearch-notification.yml | 6 ------ 2 files changed, 16 deletions(-) diff --git a/opsfiles/add-keystore-credentials-errand.yml b/opsfiles/add-keystore-credentials-errand.yml index e4c6d57..b865f3f 100644 --- a/opsfiles/add-keystore-credentials-errand.yml +++ b/opsfiles/add-keystore-credentials-errand.yml @@ -49,16 +49,6 @@ value: <<: *add-credential-map -- type: replace - path: /instance_groups/name=archiver/jobs/- - value: - <<: *add-keystore-credentials-errand - -- type: replace - path: /instance_groups/name=archiver/jobs/name=opensearch/properties/opensearch?/notifications? - value: - <<: *add-credential-map - - type: replace path: /instance_groups/name=ingestor/jobs/- value: diff --git a/opsfiles/opensearch-notification.yml b/opsfiles/opensearch-notification.yml index f4fb92a..dd592f6 100644 --- a/opsfiles/opensearch-notification.yml +++ b/opsfiles/opensearch-notification.yml @@ -27,12 +27,6 @@ defaultkeystorepass: ((default_keystore_password)) -- type: replace - path: /instance_groups/name=archiver/jobs/name=opensearch/properties/opensearch/notification? - value: - keystorepass: ((keystore_password)) - defaultkeystorepass: ((default_keystore_password)) - - type: replace path: /instance_groups/name=ingestor/jobs/name=opensearch/properties/opensearch/notification? From 6964629d9aa41a656be59ead917fa0e3684ee99d Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 13:36:12 -0400 Subject: [PATCH 005/165] missed this one --- opensearch-base.yml | 8 ++++---- opsfiles/enable-proxy-auth.yml | 5 ----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index daa51f9..f414205 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -231,10 +231,10 @@ instance_groups: bucket: ((cloudwatch_bucket)) s3_metric: bucket: ((metric_stream_bucket)) - # tls: - # use_tls: true - # certificate: ((archiver_syslog_client_tls.certificate)) - # private_key: ((archiver_syslog_client_tls.private_key)) + tls: + use_tls: true + certificate: ((archiver_syslog_client_tls.certificate)) + private_key: ((archiver_syslog_client_tls.private_key)) release: opensearch - name: upload_opensearch_config consumes: *consumes-opensearch-manager diff --git a/opsfiles/enable-proxy-auth.yml b/opsfiles/enable-proxy-auth.yml index 8950a3f..d2ee7f6 100644 --- a/opsfiles/enable-proxy-auth.yml +++ b/opsfiles/enable-proxy-auth.yml @@ -29,11 +29,6 @@ path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties?/opensearch_dashboards?/opensearch?/enable_proxy_auth value: true -# archiver -- type: replace - path: /instance_groups/name=archiver/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth - value: true - # ingestor - type: replace path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth From 1c6e1448a8a4335535ab3edffbea27345b2db1c3 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 14:12:53 -0400 Subject: [PATCH 006/165] same profile as opensearch logs --- opensearch-base.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index f414205..ad59481 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -68,7 +68,7 @@ instance_groups: azs: [z1,z2] vm_type: t3.large\ vm_extensions: - - logs-platform-profile + - logs-opensearch-profile networks: - name: services update: @@ -110,7 +110,7 @@ instance_groups: stemcell: default azs: [z1,z2] vm_extensions: - - logs-platform-profile + - logs-opensearch-profile vm_type: t3.large networks: - name: services @@ -130,7 +130,7 @@ instance_groups: - name: opensearch_manager instances: 3 vm_extensions: - - logs-platform-profile + - logs-opensearch-profile - 15GB_ephemeral_disk jobs: - name: bpm @@ -433,7 +433,7 @@ instance_groups: release: opensearch vm_extensions: - 15GB_ephemeral_disk - - logs-platform-profile + - logs-opensearch-profile stemcell: default azs: [z1] networks: From 316fa2ab49949af24f4a180a0a4c98e8f98c4135 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 14:16:31 -0400 Subject: [PATCH 007/165] fixing ingestor profile --- opensearch-base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index ad59481..4e79fd4 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -175,7 +175,7 @@ instance_groups: instances: 1 vm_extensions: - 20GB_ephemeral_disk - - logs-platform-ingestor-profile + - logs-opensearch-ingestor-profile jobs: - name: bpm release: bpm @@ -386,7 +386,7 @@ instance_groups: azs: [z1,z2] vm_type: t3.medium vm_extensions: - - logs-platform-ingestor-profile + - logs-opensearch-ingestor-profile - 20GB_ephemeral_disk networks: - name: services From 211819d9590cca01b3bad2a8144f37a24c6eab00 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 18 Mar 2026 14:31:19 -0400 Subject: [PATCH 008/165] lowering smoketests --- opensearch-base.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 4e79fd4..d222344 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -198,12 +198,6 @@ instance_groups: name: smoke_tests properties: smoke_tests: - cloudwatch_count_test: - minimum: 2 - metric_count_test: - minimum: 2 - audit_count_test: - minimum: 2 count_test: index_pattern: logs-app-* minimum: 10 @@ -212,11 +206,6 @@ instance_groups: time_field: '@timestamp' time_interval: 5m index: logs-app - log_group: ((smoketest_log_group)) - metric_index: logs-metrics- - org_guid: ((smoketest_org_guid)) - space_guid: ((smoketest_space_guid)) - rds_instance: ((smoketest_rds)) opensearch: ssl: ca: ((opensearch_node.ca)) @@ -225,12 +214,6 @@ instance_groups: s3: environment: development region: ((region)) - s3_audit: - bucket: ((audit_bucket)) - s3_cloudwatch: - bucket: ((cloudwatch_bucket)) - s3_metric: - bucket: ((metric_stream_bucket)) tls: use_tls: true certificate: ((archiver_syslog_client_tls.certificate)) From eb7b962180929119bac53c223540fe9b3323a518 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 23 Mar 2026 11:33:58 -0400 Subject: [PATCH 009/165] removing metric --- opensearch-base.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index d222344..aff581f 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -224,7 +224,6 @@ instance_groups: properties: opensearch_config: alias: ((alias)) - metric_alias: ((metric_alias)) component_templates: - shards-and-replicas: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas.json - metric-shards-and-replica: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-metrics.json @@ -353,7 +352,6 @@ instance_groups: - localhost index: ((alias)) index_type: '%{@type}' - index_metrics: ((metric_alias)) ssl: ca: ((opensearch_node.ca)) certificate: ((logstash.certificate)) From f646504fb9fdbb4606d5cc9b9e96145fddd17e1d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 23 Mar 2026 12:38:46 -0400 Subject: [PATCH 010/165] metric will be same as app --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index aff581f..afc955b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -224,6 +224,7 @@ instance_groups: properties: opensearch_config: alias: ((alias)) + metric_alias: ((metric_alias)) component_templates: - shards-and-replicas: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas.json - metric-shards-and-replica: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-metrics.json From e46a2244be51e645b6101ea9ebf965c5a3760311 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 23 Mar 2026 13:22:50 -0400 Subject: [PATCH 011/165] removing auth proxy --- ci/pipeline.yml | 60 ++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 589ef5e..dc6914e 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -61,15 +61,15 @@ jobs: tags: [iaas] params: file: finalized-release/releases-dir-opensearch.tgz - on_failure: - put: slack - params: &slack-failure-params - text: | - :x: FAILED to build OpenSearch BOSH release - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - channel: ((slack-channel-customer-success)) - username: ((slack-username)) - icon_url: ((slack-icon-url)) + # on_failure: + # put: slack + # params: &slack-failure-params + # text: | + # :x: FAILED to build OpenSearch BOSH release + # <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + # channel: ((slack-channel-customer-success)) + # username: ((slack-username)) + # icon_url: ((slack-icon-url)) - name: deploy-opensearch-development serial_groups: [bosh-development] plan: @@ -116,21 +116,21 @@ jobs: - deploy-logs-platform-config/opsfiles/enable-node-tls.yml - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml - - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml + # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml + # - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml - deploy-logs-platform-config/opsfiles/cf-development.yml - deploy-logs-platform-config/opsfiles/opensearch-notification.yml - deploy-logs-platform-config/opsfiles/add-keystore-credentials-errand.yml vars_files: - terraform-secrets/terraform.yml - on_failure: - put: slack - params: - <<: *slack-failure-params - text: | - :x: FAILED to deploy logs-platform in development - <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + # on_failure: + # put: slack + # params: + # <<: *slack-failure-params + # text: | + # :x: FAILED to deploy logs-platform in development + # <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> resources: - name: opensearch-release-git-repo @@ -180,10 +180,10 @@ resources: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/cg-pipeline-tasks.git branch: main - - name: slack - type: slack-notification - source: - url: ((slack-webhook-url)) + # - name: slack + # type: slack-notification + # source: + # url: ((slack-webhook-url)) - name: opensearch-development-deployment type: bosh-deployment source: &bosh-params-development @@ -210,14 +210,14 @@ resource_types: repository: registry-image-resource aws_region: us-gov-west-1 tag: latest - - name: slack-notification - type: registry-image - source: - aws_access_key_id: ((ecr_aws_key)) - aws_secret_access_key: ((ecr_aws_secret)) - repository: slack-notification-resource - aws_region: us-gov-west-1 - tag: latest + # - name: slack-notification + # type: registry-image + # source: + # aws_access_key_id: ((ecr_aws_key)) + # aws_secret_access_key: ((ecr_aws_secret)) + # repository: slack-notification-resource + # aws_region: us-gov-west-1 + # tag: latest - name: bosh-deployment type: registry-image source: From 5d59fb2b615304c0d309bb7f0b73e229f23a26b8 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 10:49:57 -0400 Subject: [PATCH 012/165] this allows a test release --- ci/pipeline.yml | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index dc6914e..b98f84b 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -6,13 +6,13 @@ groups: - deploy-opensearch-development jobs: - - name: build-opensearch-release + - name: build-opensearch-test-release plan: - in_parallel: - get: release-git-repo - resource: opensearch-release-git-repo + resource: opensearch-test-release-git-repo trigger: true - - get: pipeline-tasks + - get: deploy-logs-opensearch-test-config - get: final-builds-dir-tarball resource: opensearch-final-builds-dir-tarball - get: releases-dir-tarball @@ -33,8 +33,10 @@ jobs: bundle install # run release template unit tests bundle exec rspec - - task: finalize-release - file: pipeline-tasks/finalize-bosh-release.yml + - task: create-test-release + input_mapping: + deploy-logs-opensearch-config: deploy-logs-opensearch-test-config + file: deploy-logs-opensearch-test-config/ci/create-bosh-test-release.yml tags: [iaas] params: AWS_DEFAULT_REGION: ((aws_default_region)) @@ -48,19 +50,10 @@ jobs: bucket_name: ((s3-bosh-blobstore-info.bucket_name)) credentials_source: ((s3-bosh-blobstore-info.credentials_source)) server_side_encryption: ((s3-bosh-blobstore-info.server_side_encryption)) - - in_parallel: - - put: opensearch-release - tags: [iaas] - params: - file: finalized-release/opensearch-*.tgz - - put: opensearch-final-builds-dir-tarball - tags: [iaas] - params: - file: finalized-release/final-builds-dir-opensearch.tgz - - put: opensearch-releases-dir-tarball - tags: [iaas] - params: - file: finalized-release/releases-dir-opensearch.tgz + - put: opensearch-test-release + tags: [iaas] + params: + file: finalized-release/opensearch-test-*.tgz # on_failure: # put: slack # params: &slack-failure-params @@ -77,7 +70,7 @@ jobs: - get: pipeline-tasks - get: deploy-logs-platform-config trigger: true - - get: opensearch-release + - get: opensearch-test-release trigger: true - get: opensearch-stemcell-jammy trigger: true From 3d1a455e40b7a7376de1e736b3a4b342e2d53d64 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 12:13:33 -0400 Subject: [PATCH 013/165] making platform --- opsfiles/enable-dashboard-dns.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opsfiles/enable-dashboard-dns.yml b/opsfiles/enable-dashboard-dns.yml index 2b615bc..d4ef781 100644 --- a/opsfiles/enable-dashboard-dns.yml +++ b/opsfiles/enable-dashboard-dns.yml @@ -1,7 +1,7 @@ - type: replace path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties?/aliases?/- value: - domain: opensearch_dashboards.opensearch.internal + domain: opensearch_dashboards.platform-opensearch.internal targets: - query: '*' instance_group: opensearch_dashboards From f07780c0a3d78c813343c3412174726f8784342a Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 12:16:24 -0400 Subject: [PATCH 014/165] this hsould enable the dashboard --- opensearch-base.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index afc955b..13416f4 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -286,7 +286,7 @@ instance_groups: - {type: visualization, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/visualization/App-*.json"} - {type: dashboard, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/dashboard/App-*.json"} opensearch_dashboards: - host: opensearch_dashboards.opensearch.internal + host: opensearch_dashboards.platform-opensearch.internal stemcell: default azs: [z1,z2] vm_type: t3.large @@ -413,6 +413,24 @@ instance_groups: tenants: enable_private: false release: opensearch + - name: oauth2-proxy + release: oauth2-proxy + properties: + address: http://127.0.0.1:5601 + upstream: http://127.0.0.1:5602 + provider: oidc + client_id: ((client_id)) + client_secret: ((client_secret)) + cookie_secret: ((cookie_secret)) + oidc_issuer_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) + email_domain: gsa.gov + - name: secureproxy + release: secureproxy + properties: + secureproxy: + listen_port: 5600 + proxy_port: 5601 vm_extensions: - 15GB_ephemeral_disk - logs-opensearch-profile From 29cee60f1b8dfe90b70f711ca0c626b5d69711c3 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 12:18:26 -0400 Subject: [PATCH 015/165] this hsould enable the dashboard --- opensearch-base.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 13416f4..c651e4a 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -18,6 +18,8 @@ releases: - {name: jammy-snort, version: latest} - {name: bosh-dns-aliases, version: latest} - {name: bpm, version: latest} +- {name: oauth2-proxy, version: latest} +- {name: secureproxy, version: latest} stemcells: - alias: default From 614b3a65cf9101df71d5011858227424fd426afe Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 13:20:08 -0400 Subject: [PATCH 016/165] making less --- opensearch-scaling-development.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 199ba67..5b66be9 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 3 + value: 0 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? @@ -25,7 +25,7 @@ - type: replace path: /instance_groups/name=opensearch_old_data?/instances? - value: 2 + value: 0 - type: replace path: /instance_groups/name=opensearch_old_data?/vm_type? From 9985a149a8ea6267d3c91e0901dea82a0ba98c77 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 14:05:24 -0400 Subject: [PATCH 017/165] fight of the dashbaords --- opensearch-base.yml | 4 ++-- opsfiles/enable-proxy-auth.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index c651e4a..861667e 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -418,7 +418,7 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5601 + address: http://127.0.0.1:5605 upstream: http://127.0.0.1:5602 provider: oidc client_id: ((client_id)) @@ -432,7 +432,7 @@ instance_groups: properties: secureproxy: listen_port: 5600 - proxy_port: 5601 + proxy_port: 5605 vm_extensions: - 15GB_ephemeral_disk - logs-opensearch-profile diff --git a/opsfiles/enable-proxy-auth.yml b/opsfiles/enable-proxy-auth.yml index d2ee7f6..b02511a 100644 --- a/opsfiles/enable-proxy-auth.yml +++ b/opsfiles/enable-proxy-auth.yml @@ -78,7 +78,7 @@ ca: /var/vcap/jobs/opensearch_dashboards/config/ssl/opensearch.ca certificate: /var/vcap/jobs/opensearch_dashboards/config/ssl/dashboard-web.crt key: /var/vcap/jobs/opensearch_dashboards/config/ssl/dashboard-web.key - url: https://localhost:5601 + url: https://localhost:5605 port: 8000 secret_key: ((auth-proxy-secret-key)) uaa: From 238213d5ed57a96b79d93327c4612384b77c24cf Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 14:06:43 -0400 Subject: [PATCH 018/165] should be 5605 --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 861667e..42a8f3f 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -404,6 +404,7 @@ instance_groups: config_options: server.maxPayloadBytes: 4194304 console.enabled: false + port: 5605 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production From 91464c76c6f21110d4b2c79dcb136482e58693eb Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 14:45:15 -0400 Subject: [PATCH 019/165] should be 5605 --- opensearch-base.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 42a8f3f..e11bbf1 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -404,7 +404,7 @@ instance_groups: config_options: server.maxPayloadBytes: 4194304 console.enabled: false - port: 5605 + port: 5604 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production @@ -419,8 +419,8 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5605 - upstream: http://127.0.0.1:5602 + address: http://127.0.0.1:5603 + upstream: http://127.0.0.1:5604 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) @@ -432,8 +432,8 @@ instance_groups: release: secureproxy properties: secureproxy: - listen_port: 5600 - proxy_port: 5605 + listen_port: 5602 + proxy_port: 5603 vm_extensions: - 15GB_ephemeral_disk - logs-opensearch-profile From f64debfb5715be780f9b3e8a6d356b24ff433fd5 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 14:55:40 -0400 Subject: [PATCH 020/165] should be 5605 --- opensearch-base.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index e11bbf1..c07b671 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -404,7 +404,7 @@ instance_groups: config_options: server.maxPayloadBytes: 4194304 console.enabled: false - port: 5604 + port: 5602 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production @@ -419,8 +419,8 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5603 - upstream: http://127.0.0.1:5604 + address: http://127.0.0.1:5601 + upstream: http://127.0.0.1:5602 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) @@ -432,8 +432,8 @@ instance_groups: release: secureproxy properties: secureproxy: - listen_port: 5602 - proxy_port: 5603 + listen_port: 5600 + proxy_port: 5601 vm_extensions: - 15GB_ephemeral_disk - logs-opensearch-profile From 17cccdce76df542a9cf5745cb20ca5c7a34970ff Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 15:29:28 -0400 Subject: [PATCH 021/165] should be 5605 --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 5b66be9..b426a05 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From 947d2be7c7ee64a0febff8e988a30c072293754c Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 16:00:52 -0400 Subject: [PATCH 022/165] takes 2 to be green --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index b426a05..b43da45 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 1 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From 1c366c54b0c617918b91163b7a90ea1ba7920fb8 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 24 Mar 2026 16:11:48 -0400 Subject: [PATCH 023/165] disable post to make them exist --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index c07b671..582d73d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -99,7 +99,7 @@ instance_groups: fd: 131072 # 2 ** 17 health: timeout: 900 - disable_post_start: false + disable_post_start: true recovery: delay_allocation_restart: "10m" config_options: From 62e63bf364070547873cb7e5d91c553892fc3a86 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 08:09:29 -0400 Subject: [PATCH 024/165] adding in correction --- opsfiles/enable-dashboards-tls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index 04ff662..941e4dc 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -18,7 +18,7 @@ common_name: opensearch_dashboard.web alternative_names: - localhost - - opensearch_dashboards.opensearch.internal + - opensearch_dashboards.platform-opensearch.internal extended_key_usage: - server_auth - client_auth From 19129a7f7f9c33d836f0213db06a72cc42617d38 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 09:12:10 -0400 Subject: [PATCH 025/165] giving unique name --- opsfiles/enable-dashboards-tls.yml | 10 +++++----- opsfiles/enable-node-tls.yml | 16 ++++++++-------- opsfiles/enable-proxy-auth.yml | 2 +- opsfiles/enable-syslog.yml | 12 ++++++------ 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index 941e4dc..70066e4 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -3,18 +3,18 @@ path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties?/opensearch_dashboards?/server?/ssl? value: enabled: true - certificate: ((opensearch_dashboard_web.certificate)) - private_key: ((opensearch_dashboard_web.private_key)) + certificate: ((platform_dashboard_web.certificate)) + private_key: ((platform_dashboard_web.private_key)) # variables - type: replace - path: /variables/name=opensearch_dashboard_web? + path: /variables/name=platform_dashboard_web? value: - name: opensearch_dashboard_web + name: platform_dashboard_web type: certificate update_mode: converge options: - ca: opensearch_ca + ca: platform_opensearch_ca common_name: opensearch_dashboard.web alternative_names: - localhost diff --git a/opsfiles/enable-node-tls.yml b/opsfiles/enable-node-tls.yml index 855da4f..56bcaa7 100644 --- a/opsfiles/enable-node-tls.yml +++ b/opsfiles/enable-node-tls.yml @@ -125,13 +125,13 @@ # variables - type: replace - path: /variables/name=opensearch_ca? + path: /variables/name=platform_opensearch_ca? value: - name: opensearch_ca + name: platform_opensearch_ca type: certificate options: is_ca: true - common_name: opensearch-ca + common_name: platform-opensearch-ca duration: 1825 # 365 days * 5 years - type: replace @@ -143,7 +143,7 @@ options: alternative_names: - localhost - ca: opensearch_ca + ca: platform_opensearch_ca common_name: logstash.opensearch.internal extended_key_usage: - server_auth @@ -156,7 +156,7 @@ type: certificate update_mode: converge options: - ca: opensearch_ca + ca: platform_opensearch_ca common_name: node.opensearch.internal alternative_names: ["localhost"] extended_key_usage: @@ -170,7 +170,7 @@ type: certificate update_mode: converge options: - ca: opensearch_ca + ca: platform_opensearch_ca common_name: dashboard.opensearch.internal alternative_names: ["localhost"] extended_key_usage: @@ -184,7 +184,7 @@ type: certificate update_mode: converge options: - ca: opensearch_ca + ca: platform_opensearch_ca common_name: smoketest.opensearch.internal alternative_names: ["localhost"] extended_key_usage: @@ -197,7 +197,7 @@ name: opensearch_admin type: certificate options: - ca: opensearch_ca + ca: platform_opensearch_ca common_name: admin.opensearch.internal extended_key_usage: - client_auth diff --git a/opsfiles/enable-proxy-auth.yml b/opsfiles/enable-proxy-auth.yml index b02511a..7a8a96e 100644 --- a/opsfiles/enable-proxy-auth.yml +++ b/opsfiles/enable-proxy-auth.yml @@ -43,7 +43,7 @@ alternative_names: - localhost - auth-proxy.opensearch.internal - ca: opensearch_ca + ca: platform_opensearch_ca common_name: auth-proxy.opensearch.internal extended_key_usage: - server_auth diff --git a/opsfiles/enable-syslog.yml b/opsfiles/enable-syslog.yml index 0e2e0f6..df4797e 100644 --- a/opsfiles/enable-syslog.yml +++ b/opsfiles/enable-syslog.yml @@ -7,7 +7,7 @@ options: alternative_names: - logs-platform.ingestor.service.cf.internal - ca: opensearch_ca + ca: platform_opensearch_ca common_name: logs-platform.ingestor.service.cf.internal extended_key_usage: - client_auth @@ -21,7 +21,7 @@ options: alternative_names: - logs-platform.ingestor.service.cf.internal - ca: opensearch_ca + ca: platform_opensearch_ca common_name: logs-platform.ingestor.service.cf.internal extended_key_usage: - server_auth @@ -35,7 +35,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: opensearch_ca + ca: platform_opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - client_auth @@ -49,7 +49,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: opensearch_ca + ca: platform_opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - server_auth @@ -63,7 +63,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: opensearch_ca + ca: platform_opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - client_auth @@ -77,7 +77,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: opensearch_ca + ca: platform_opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - server_auth From ceeffa071dea8f132d44a52b559822ba54f76306 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 09:27:50 -0400 Subject: [PATCH 026/165] makign manager first --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index b43da45..5b66be9 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 0 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From 524f04948b46b46960a6e38914d3cb055253e384 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 10:38:31 -0400 Subject: [PATCH 027/165] makign manager first --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 5b66be9..b43da45 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From 5f556b22b1c2b3015d20e5bfcf3d7cb272398abe Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 10:57:15 -0400 Subject: [PATCH 028/165] makign manager first --- ci/pipeline.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index b98f84b..cfd148d 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -2,7 +2,7 @@ groups: - name: development jobs: - - build-opensearch-release + - build-opensearch-test-release - deploy-opensearch-development jobs: @@ -12,7 +12,7 @@ jobs: - get: release-git-repo resource: opensearch-test-release-git-repo trigger: true - - get: deploy-logs-opensearch-test-config + - get: deploy-logs-platform-config - get: final-builds-dir-tarball resource: opensearch-final-builds-dir-tarball - get: releases-dir-tarball @@ -35,8 +35,8 @@ jobs: bundle exec rspec - task: create-test-release input_mapping: - deploy-logs-opensearch-config: deploy-logs-opensearch-test-config - file: deploy-logs-opensearch-test-config/ci/create-bosh-test-release.yml + deploy-logs-opensearch-config: deploy-logs-platform-config + file: deploy-logs-platform-config/ci/create-bosh-test-release.yml tags: [iaas] params: AWS_DEFAULT_REGION: ((aws_default_region)) @@ -99,10 +99,11 @@ jobs: file: deploy-logs-platform-config/ci/terraform-secrets.yml - put: opensearch-development-deployment params: + skip_drain: true manifest: opensearch-manifest/manifest.yml - dry_run: true + # dry_run: true releases: - - opensearch-release/*.tgz + - opensearch-test-release/*.tgz stemcells: - opensearch-stemcell-jammy/*.tgz ops_files: @@ -126,12 +127,13 @@ jobs: # <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> resources: - - name: opensearch-release-git-repo + - name: opensearch-test-release-git-repo type: git source: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/opensearch-boshrelease branch: platform + - name: opensearch-final-builds-dir-tarball type: s3-iam source: &bosh-releases-bucket-info @@ -139,18 +141,21 @@ resources: region_name: ((aws-region)) server_side_encryption: AES256 versioned_file: final-builds-dir-opensearch.tgz + - name: opensearch-releases-dir-tarball type: s3-iam source: <<: *bosh-releases-bucket-info versioned_file: releases-dir-opensearch.tgz - - name: opensearch-release + + - name: opensearch-test-release type: s3-iam source: bucket: ((s3-bosh-releases-bucket)) region_name: ((aws-region)) - regexp: opensearch-([\d\.]*).tgz + regexp: opensearch-test-([\d\.]*).tgz server_side_encryption: AES256 + - name: terraform-yaml-development type: s3-iam source: From 2ec6e082570de972a6c34f31b9d3828f70140a6b Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 11:02:06 -0400 Subject: [PATCH 029/165] making CA opensearch again --- opensearch-scaling-development.yml | 2 +- opsfiles/enable-dashboards-tls.yml | 2 +- opsfiles/enable-node-tls.yml | 14 +++++++------- opsfiles/enable-proxy-auth.yml | 2 +- opsfiles/enable-syslog.yml | 12 ++++++------ 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index b43da45..5b66be9 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 0 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index 70066e4..09f2267 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -14,7 +14,7 @@ type: certificate update_mode: converge options: - ca: platform_opensearch_ca + ca: opensearch_ca common_name: opensearch_dashboard.web alternative_names: - localhost diff --git a/opsfiles/enable-node-tls.yml b/opsfiles/enable-node-tls.yml index 56bcaa7..3e5a044 100644 --- a/opsfiles/enable-node-tls.yml +++ b/opsfiles/enable-node-tls.yml @@ -125,9 +125,9 @@ # variables - type: replace - path: /variables/name=platform_opensearch_ca? + path: /variables/name=opensearch_ca? value: - name: platform_opensearch_ca + name: opensearch_ca type: certificate options: is_ca: true @@ -143,7 +143,7 @@ options: alternative_names: - localhost - ca: platform_opensearch_ca + ca: opensearch_ca common_name: logstash.opensearch.internal extended_key_usage: - server_auth @@ -156,7 +156,7 @@ type: certificate update_mode: converge options: - ca: platform_opensearch_ca + ca: opensearch_ca common_name: node.opensearch.internal alternative_names: ["localhost"] extended_key_usage: @@ -170,7 +170,7 @@ type: certificate update_mode: converge options: - ca: platform_opensearch_ca + ca: opensearch_ca common_name: dashboard.opensearch.internal alternative_names: ["localhost"] extended_key_usage: @@ -184,7 +184,7 @@ type: certificate update_mode: converge options: - ca: platform_opensearch_ca + ca: opensearch_ca common_name: smoketest.opensearch.internal alternative_names: ["localhost"] extended_key_usage: @@ -197,7 +197,7 @@ name: opensearch_admin type: certificate options: - ca: platform_opensearch_ca + ca: opensearch_ca common_name: admin.opensearch.internal extended_key_usage: - client_auth diff --git a/opsfiles/enable-proxy-auth.yml b/opsfiles/enable-proxy-auth.yml index 7a8a96e..b02511a 100644 --- a/opsfiles/enable-proxy-auth.yml +++ b/opsfiles/enable-proxy-auth.yml @@ -43,7 +43,7 @@ alternative_names: - localhost - auth-proxy.opensearch.internal - ca: platform_opensearch_ca + ca: opensearch_ca common_name: auth-proxy.opensearch.internal extended_key_usage: - server_auth diff --git a/opsfiles/enable-syslog.yml b/opsfiles/enable-syslog.yml index df4797e..0e2e0f6 100644 --- a/opsfiles/enable-syslog.yml +++ b/opsfiles/enable-syslog.yml @@ -7,7 +7,7 @@ options: alternative_names: - logs-platform.ingestor.service.cf.internal - ca: platform_opensearch_ca + ca: opensearch_ca common_name: logs-platform.ingestor.service.cf.internal extended_key_usage: - client_auth @@ -21,7 +21,7 @@ options: alternative_names: - logs-platform.ingestor.service.cf.internal - ca: platform_opensearch_ca + ca: opensearch_ca common_name: logs-platform.ingestor.service.cf.internal extended_key_usage: - server_auth @@ -35,7 +35,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: platform_opensearch_ca + ca: opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - client_auth @@ -49,7 +49,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: platform_opensearch_ca + ca: opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - server_auth @@ -63,7 +63,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: platform_opensearch_ca + ca: opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - client_auth @@ -77,7 +77,7 @@ options: alternative_names: - logs-platform.archiver.service.cf.internal - ca: platform_opensearch_ca + ca: opensearch_ca common_name: logs-platform.archiver.service.cf.internal extended_key_usage: - server_auth From 98eb6added176b988e1921fddb17ed53e33a746f Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 11:37:30 -0400 Subject: [PATCH 030/165] hopefully fix dashboard --- opsfiles/enable-dashboard-dns.yml | 2 +- opsfiles/enable-dashboards-tls.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opsfiles/enable-dashboard-dns.yml b/opsfiles/enable-dashboard-dns.yml index d4ef781..2b615bc 100644 --- a/opsfiles/enable-dashboard-dns.yml +++ b/opsfiles/enable-dashboard-dns.yml @@ -1,7 +1,7 @@ - type: replace path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties?/aliases?/- value: - domain: opensearch_dashboards.platform-opensearch.internal + domain: opensearch_dashboards.opensearch.internal targets: - query: '*' instance_group: opensearch_dashboards diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index 09f2267..cd0a7e8 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -18,7 +18,7 @@ common_name: opensearch_dashboard.web alternative_names: - localhost - - opensearch_dashboards.platform-opensearch.internal + - opensearch_dashboards.opensearch.internal extended_key_usage: - server_auth - client_auth From 214ae9e7164d82f8a94e98d47a3f67f826335014 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 11:59:58 -0400 Subject: [PATCH 031/165] need data nodes --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 5b66be9..b43da45 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From 615c4a5b4d4b7c6f414132deb70b4c60b152de90 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 25 Mar 2026 12:26:29 -0400 Subject: [PATCH 032/165] adding in post --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 582d73d..c07b671 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -99,7 +99,7 @@ instance_groups: fd: 131072 # 2 ** 17 health: timeout: 900 - disable_post_start: true + disable_post_start: false recovery: delay_allocation_restart: "10m" config_options: From bc8bcd7efcd1ccdeb3418785ed8d72069fcf9f27 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 26 Mar 2026 10:17:44 -0400 Subject: [PATCH 033/165] connecting to right port --- opensearch-base.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index c07b671..03cead0 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -404,7 +404,7 @@ instance_groups: config_options: server.maxPayloadBytes: 4194304 console.enabled: false - port: 5602 + port: 5607 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production @@ -419,8 +419,8 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5601 - upstream: http://127.0.0.1:5602 + address: http://127.0.0.1:5606 + upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) @@ -432,8 +432,8 @@ instance_groups: release: secureproxy properties: secureproxy: - listen_port: 5600 - proxy_port: 5601 + listen_port: 5605 + proxy_port: 5606 vm_extensions: - 15GB_ephemeral_disk - logs-opensearch-profile From 1479767fa6dd13829cbccb743366c1fa9e7faa47 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 26 Mar 2026 13:22:29 -0400 Subject: [PATCH 034/165] fixing --- ci/pipeline.yml | 3 +-- opsfiles/enable-auth-proxy-route-dev.yml | 25 +++++++++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index cfd148d..7e7ff3f 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -99,7 +99,6 @@ jobs: file: deploy-logs-platform-config/ci/terraform-secrets.yml - put: opensearch-development-deployment params: - skip_drain: true manifest: opensearch-manifest/manifest.yml # dry_run: true releases: @@ -111,7 +110,7 @@ jobs: - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - # - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml + - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml - deploy-logs-platform-config/opsfiles/cf-development.yml - deploy-logs-platform-config/opsfiles/opensearch-notification.yml diff --git a/opsfiles/enable-auth-proxy-route-dev.yml b/opsfiles/enable-auth-proxy-route-dev.yml index c9e3ea5..7b698e2 100644 --- a/opsfiles/enable-auth-proxy-route-dev.yml +++ b/opsfiles/enable-auth-proxy-route-dev.yml @@ -1,3 +1,23 @@ + +# add route for auth proxy job +- type: replace + path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar? + value: + consumes: + nats-tls: + from: nats-tls + name: route_registrar + properties: + route_registrar: + routes: + - name: opensearch-auth-proxy + registration_interval: 2s + server_cert_domain_san: auth-proxy.opensearch.internal + timeout: 1s + tls_port: 8000 + uris: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov + release: routing + # add nats info for auth proxy route - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/nats?/tls? @@ -6,11 +26,6 @@ client_key: ((/bosh/cf-development/nats_client_cert.private_key)) enabled: true -# add address for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/route_registrar?/routes?/name=opensearch-auth-proxy/uris?/- - value: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov - - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/consumes?/nats-tls?/deployment? value: cf-development From 0288d9332b81d15459c5a658732ebcb70f7a48a3 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 26 Mar 2026 13:49:33 -0400 Subject: [PATCH 035/165] fixing --- opsfiles/enable-auth-proxy-route-dev.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opsfiles/enable-auth-proxy-route-dev.yml b/opsfiles/enable-auth-proxy-route-dev.yml index 7b698e2..8d2f8f7 100644 --- a/opsfiles/enable-auth-proxy-route-dev.yml +++ b/opsfiles/enable-auth-proxy-route-dev.yml @@ -15,7 +15,8 @@ server_cert_domain_san: auth-proxy.opensearch.internal timeout: 1s tls_port: 8000 - uris: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov + uris: + - logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov release: routing # add nats info for auth proxy route From 7b00f749f1566a049cc2ef4eff9fd7c92ba8a1dc Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 26 Mar 2026 14:38:42 -0400 Subject: [PATCH 036/165] fixing --- opensearch-base.yml | 12 ++++++++++++ opsfiles/enable-auth-proxy-route-dev.yml | 21 --------------------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 03cead0..31601c6 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -434,6 +434,18 @@ instance_groups: secureproxy: listen_port: 5605 proxy_port: 5606 + - name: route_registrar + release: route-registrar + consumes: + nats-tls: + from: nats-tls + properties: + route_registrar: + routes: + - name: platform-auth-proxy + port: 5605 + uri: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov + release: routing vm_extensions: - 15GB_ephemeral_disk - logs-opensearch-profile diff --git a/opsfiles/enable-auth-proxy-route-dev.yml b/opsfiles/enable-auth-proxy-route-dev.yml index 8d2f8f7..897857e 100644 --- a/opsfiles/enable-auth-proxy-route-dev.yml +++ b/opsfiles/enable-auth-proxy-route-dev.yml @@ -1,24 +1,3 @@ - -# add route for auth proxy job -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar? - value: - consumes: - nats-tls: - from: nats-tls - name: route_registrar - properties: - route_registrar: - routes: - - name: opensearch-auth-proxy - registration_interval: 2s - server_cert_domain_san: auth-proxy.opensearch.internal - timeout: 1s - tls_port: 8000 - uris: - - logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov - release: routing - # add nats info for auth proxy route - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/nats?/tls? From 8d51155bbf358db734f93b1dbad5887a8e479d4d Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 26 Mar 2026 18:49:50 -0400 Subject: [PATCH 037/165] this should be enabled --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 31601c6..8e8224f 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -444,6 +444,7 @@ instance_groups: routes: - name: platform-auth-proxy port: 5605 + tls_port: 8000 uri: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov release: routing vm_extensions: From 97dad7c0683a0991c6a44960f01d053422f1d678 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 08:32:02 -0400 Subject: [PATCH 038/165] attempting to make work --- opensearch-base.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 8e8224f..31601c6 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -444,7 +444,6 @@ instance_groups: routes: - name: platform-auth-proxy port: 5605 - tls_port: 8000 uri: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov release: routing vm_extensions: From 6e978343acdba2d545ae09a185beb18f2ed3ba74 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 08:35:20 -0400 Subject: [PATCH 039/165] removing so it can succeed --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index b43da45..677720a 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -41,7 +41,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=opensearch_dashboards?/vm_type? From 037ac57511788cc09aed0dfe8af1ab3add3f83fb Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 08:53:03 -0400 Subject: [PATCH 040/165] enabling for testing --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 677720a..b43da45 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -41,7 +41,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=opensearch_dashboards?/vm_type? From 04d5c033afd6d9020f64b32cee2061755a7c2d94 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 09:12:11 -0400 Subject: [PATCH 041/165] enabling for testing --- opensearch-base.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 31601c6..71e0ecb 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -444,7 +444,9 @@ instance_groups: routes: - name: platform-auth-proxy port: 5605 - uri: logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov + registration_interval: 2s + uris: + - logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov release: routing vm_extensions: - 15GB_ephemeral_disk From 37334e18515f55207214c9d390842296f7f82d3f Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 09:36:30 -0400 Subject: [PATCH 042/165] enabling for testing --- opensearch-base.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 71e0ecb..b6a5a0d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -404,6 +404,9 @@ instance_groups: config_options: server.maxPayloadBytes: 4194304 console.enabled: false + server: + ssl: + enabled: false port: 5607 defaultAppId: dashboard/App-Overview env: From 6e35b6f0a36c37ae9bdb84d25ebc49be6a6ab672 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 09:48:20 -0400 Subject: [PATCH 043/165] removing --- ci/pipeline.yml | 2 +- opsfiles/enable-dashboards-tls.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 7e7ff3f..9269dec 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -108,7 +108,7 @@ jobs: ops_files: - deploy-logs-platform-config/opsfiles/enable-node-tls.yml - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + #- deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index cd0a7e8..b17c5d0 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -5,7 +5,7 @@ enabled: true certificate: ((platform_dashboard_web.certificate)) private_key: ((platform_dashboard_web.private_key)) - + # variables - type: replace path: /variables/name=platform_dashboard_web? From 2c5c21d010c626ff4c5b02de7341442b67b736b0 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 09:55:58 -0400 Subject: [PATCH 044/165] removing enabling but making better --- ci/pipeline.yml | 2 +- opsfiles/enable-dashboards-tls.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 9269dec..7e7ff3f 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -108,7 +108,7 @@ jobs: ops_files: - deploy-logs-platform-config/opsfiles/enable-node-tls.yml - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - #- deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index b17c5d0..514a11b 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -2,7 +2,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties?/opensearch_dashboards?/server?/ssl? value: - enabled: true + enabled: false certificate: ((platform_dashboard_web.certificate)) private_key: ((platform_dashboard_web.private_key)) From f93d8e5888baad1bcc498b5e70f1e8a1c96bce5e Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 10:12:22 -0400 Subject: [PATCH 045/165] updating to use https --- opensearch-base.yml | 2 +- opsfiles/enable-dashboards-tls.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index b6a5a0d..cbd892e 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -423,7 +423,7 @@ instance_groups: release: oauth2-proxy properties: address: http://127.0.0.1:5606 - upstream: http://127.0.0.1:5607 + upstream: https://127.0.0.1:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index 514a11b..b17c5d0 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -2,7 +2,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties?/opensearch_dashboards?/server?/ssl? value: - enabled: false + enabled: true certificate: ((platform_dashboard_web.certificate)) private_key: ((platform_dashboard_web.private_key)) From fd8bfd06c779b4d7c7a765ffce698ac808ba945f Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 27 Mar 2026 11:08:48 -0400 Subject: [PATCH 046/165] updating to use https --- opensearch-base.yml | 2 +- opsfiles/enable-dashboards-tls.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index cbd892e..b6a5a0d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -423,7 +423,7 @@ instance_groups: release: oauth2-proxy properties: address: http://127.0.0.1:5606 - upstream: https://127.0.0.1:5607 + upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index b17c5d0..514a11b 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -2,7 +2,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties?/opensearch_dashboards?/server?/ssl? value: - enabled: true + enabled: false certificate: ((platform_dashboard_web.certificate)) private_key: ((platform_dashboard_web.private_key)) From fabf0f138059193c34193574a2ccb0ea8426d4dc Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 30 Mar 2026 09:36:06 -0400 Subject: [PATCH 047/165] disable post --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index b6a5a0d..b784711 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -99,7 +99,7 @@ instance_groups: fd: 131072 # 2 ** 17 health: timeout: 900 - disable_post_start: false + disable_post_start: true recovery: delay_allocation_restart: "10m" config_options: From 82540b7a0c0eef9b405b59b095682e6833af26a1 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 30 Mar 2026 11:18:40 -0400 Subject: [PATCH 048/165] the order is better to have the manager first --- opensearch-base.yml | 81 +++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index b784711..29dd784 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -28,6 +28,47 @@ stemcells: instance_groups: + +- name: opensearch_manager + instances: 3 + vm_extensions: + - logs-opensearch-profile + - 15GB_ephemeral_disk + jobs: + - name: bpm + release: bpm + - name: opensearch + consumes: *consumes-opensearch-manager + provides: + opensearch: + as: opensearch_manager + properties: + opensearch: + clustername: opensearch + limits: + fd: 131072 # 2 ** 17 + jvm_options: + - "-Dlog4j2.formatMsgNoLookups=true" + node: + allow_cluster_manager: true + allow_data: false + release: opensearch + - name: snort-config + properties: + snort: + rules: + - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"POST"; http_method; content: "logs-platform-app"; http_uri; content:"/_update"; http_uri; classtype:web-application-attack; sid:343080002; rev:1;)' + - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"DELETE"; http_method; content: "logs-platform-app"; http_uri; classtype:web-application-attack; sid:343080004; rev:1;)' + release: jammy-snort + azs: [z1,z2] + persistent_disk_type: logs_opensearch_os_master + stemcell: default + vm_type: t3.large + networks: + - name: services + update: + serial: true # Block on this job to create deploy group 2 + ####################################################### #First deploy group - opensearch_data # Data nodes are updated first as they are the most flexible with ingestors and need to be complete before managers. @@ -129,45 +170,7 @@ instance_groups: # They are updated so one is always up as opensearch can survive on one manager in a disaster ######################################################### -- name: opensearch_manager - instances: 3 - vm_extensions: - - logs-opensearch-profile - - 15GB_ephemeral_disk - jobs: - - name: bpm - release: bpm - - name: opensearch - consumes: *consumes-opensearch-manager - provides: - opensearch: - as: opensearch_manager - properties: - opensearch: - clustername: opensearch - limits: - fd: 131072 # 2 ** 17 - jvm_options: - - "-Dlog4j2.formatMsgNoLookups=true" - node: - allow_cluster_manager: true - allow_data: false - release: opensearch - - name: snort-config - properties: - snort: - rules: - - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"POST"; http_method; content: "logs-platform-app"; http_uri; content:"/_update"; http_uri; classtype:web-application-attack; sid:343080002; rev:1;)' - - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"DELETE"; http_method; content: "logs-platform-app"; http_uri; classtype:web-application-attack; sid:343080004; rev:1;)' - release: jammy-snort - azs: [z1,z2] - persistent_disk_type: logs_opensearch_os_master - stemcell: default - vm_type: t3.large - networks: - - name: services - update: - serial: true # Block on this job to create deploy group 2 + ######################################################### #3rd deploy group - maintenance From 9196aaff993e66869296269b6afc7588deda46ea Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 1 Apr 2026 08:31:33 -0400 Subject: [PATCH 049/165] addition of oidc variables --- opensearch-base.yml | 119 ++++++++++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 49 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 29dd784..7157678 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -28,47 +28,6 @@ stemcells: instance_groups: - -- name: opensearch_manager - instances: 3 - vm_extensions: - - logs-opensearch-profile - - 15GB_ephemeral_disk - jobs: - - name: bpm - release: bpm - - name: opensearch - consumes: *consumes-opensearch-manager - provides: - opensearch: - as: opensearch_manager - properties: - opensearch: - clustername: opensearch - limits: - fd: 131072 # 2 ** 17 - jvm_options: - - "-Dlog4j2.formatMsgNoLookups=true" - node: - allow_cluster_manager: true - allow_data: false - release: opensearch - - name: snort-config - properties: - snort: - rules: - - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"POST"; http_method; content: "logs-platform-app"; http_uri; content:"/_update"; http_uri; classtype:web-application-attack; sid:343080002; rev:1;)' - - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"DELETE"; http_method; content: "logs-platform-app"; http_uri; classtype:web-application-attack; sid:343080004; rev:1;)' - release: jammy-snort - azs: [z1,z2] - persistent_disk_type: logs_opensearch_os_master - stemcell: default - vm_type: t3.large - networks: - - name: services - update: - serial: true # Block on this job to create deploy group 2 - ####################################################### #First deploy group - opensearch_data # Data nodes are updated first as they are the most flexible with ingestors and need to be complete before managers. @@ -102,6 +61,11 @@ instance_groups: delay_allocation_restart: "15m" config_options: indices.query.bool.max_clause_count: 2048 + oidc: + enabled: true + hostname: ((oidc_host)) + client_id: ((oidc_client)) + client_secret: ((oidc_secret)) jvm_options: - "-Dlog4j2.formatMsgNoLookups=true" cf: @@ -131,6 +95,11 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + oidc: + enabled: true + hostname: ((oidc_host)) + client_id: ((oidc_client)) + client_secret: ((oidc_secret)) node: attributes: box_type: hot @@ -170,7 +139,50 @@ instance_groups: # They are updated so one is always up as opensearch can survive on one manager in a disaster ######################################################### - +- name: opensearch_manager + instances: 3 + vm_extensions: + - logs-opensearch-profile + - 15GB_ephemeral_disk + jobs: + - name: bpm + release: bpm + - name: opensearch + consumes: *consumes-opensearch-manager + provides: + opensearch: + as: opensearch_manager + properties: + opensearch: + oidc: + enabled: true + hostname: ((oidc_host)) + client_id: ((oidc_client)) + client_secret: ((oidc_secret)) + clustername: opensearch + limits: + fd: 131072 # 2 ** 17 + jvm_options: + - "-Dlog4j2.formatMsgNoLookups=true" + node: + allow_cluster_manager: true + allow_data: false + release: opensearch + - name: snort-config + properties: + snort: + rules: + - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"POST"; http_method; content: "logs-platform-app"; http_uri; content:"/_update"; http_uri; classtype:web-application-attack; sid:343080002; rev:1;)' + - 'alert tcp any any -> any 9200 (msg:"Unexpected opensearch action"; content:"DELETE"; http_method; content: "logs-platform-app"; http_uri; classtype:web-application-attack; sid:343080004; rev:1;)' + release: jammy-snort + azs: [z1,z2] + persistent_disk_type: logs_opensearch_os_master + stemcell: default + vm_type: t3.large + networks: + - name: services + update: + serial: true # Block on this job to create deploy group 2 ######################################################### #3rd deploy group - maintenance @@ -189,6 +201,11 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + oidc: + enabled: true + hostname: ((oidc_host)) + client_id: ((oidc_client)) + client_secret: ((oidc_secret)) clustername: opensearch limits: fd: 131072 # 2 ** 17 @@ -318,6 +335,11 @@ instance_groups: http_host: 127.0.0.1 jvm_options: - -Dlog4j2.formatMsgNoLookups=true + oidc: + enabled: true + hostname: ((oidc_host)) + client_id: ((oidc_client)) + client_secret: ((oidc_secret)) - name: ingestor_syslog consumes: *consumes-opensearch-manager properties: @@ -396,6 +418,11 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + oidc: + enabled: true + hostname: ((oidc_host)) + client_id: ((oidc_client)) + client_secret: ((oidc_secret)) heap_size: 1G http_host: 127.0.0.1 jvm_options: @@ -425,7 +452,7 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5606 + address: http://127.0.0.1:5605 upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) @@ -434,12 +461,6 @@ instance_groups: oidc_issuer_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) email_domain: gsa.gov - - name: secureproxy - release: secureproxy - properties: - secureproxy: - listen_port: 5605 - proxy_port: 5606 - name: route_registrar release: route-registrar consumes: From be566a1a88a4f6abd344215357dfa2b56246c206 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 1 Apr 2026 10:19:45 -0400 Subject: [PATCH 050/165] manager first --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index b43da45..5b66be9 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 0 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From 63dded07cdd6258ef626a640ac2c5d17abd3b578 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 1 Apr 2026 14:37:38 -0400 Subject: [PATCH 051/165] allowing auth for dashboard --- opsfiles/enable-dashboards-tls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opsfiles/enable-dashboards-tls.yml b/opsfiles/enable-dashboards-tls.yml index 514a11b..b17c5d0 100644 --- a/opsfiles/enable-dashboards-tls.yml +++ b/opsfiles/enable-dashboards-tls.yml @@ -2,7 +2,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties?/opensearch_dashboards?/server?/ssl? value: - enabled: false + enabled: true certificate: ((platform_dashboard_web.certificate)) private_key: ((platform_dashboard_web.private_key)) From b7b5f029421000210a6bf46b16ce9ee7e94c149c Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 1 Apr 2026 14:43:59 -0400 Subject: [PATCH 052/165] removing vms not needed yet --- opensearch-scaling-development.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 5b66be9..bfd87ec 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -57,7 +57,7 @@ - type: replace path: /instance_groups/name=ingestor?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=ingestor?/vm_type? @@ -96,7 +96,7 @@ - type: replace path: /instance_groups/name=maintenance?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=maintenance?/vm_type? From de2754a4d31715d7a26a63d4d075fd8b7dc5a28e Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 1 Apr 2026 14:51:39 -0400 Subject: [PATCH 053/165] allowing auth for dashboard --- opensearch-base.yml | 2 +- opensearch-scaling-development.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 7157678..b6507d9 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -56,7 +56,7 @@ instance_groups: fd: 131072 # 2 ** 17 health: timeout: 900 - disable_post_start: false + disable_post_start: true recovery: delay_allocation_restart: "15m" config_options: diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index bfd87ec..bb874c1 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From 4cc1c78da53d4906695ed0db72675260768a5d47 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 2 Apr 2026 10:05:39 -0400 Subject: [PATCH 054/165] putting secure back in --- opensearch-base.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index b6507d9..b988e2f 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -449,10 +449,16 @@ instance_groups: tenants: enable_private: false release: opensearch + - name: secureproxy + release: secureproxy + properties: + secureproxy: + listen_port: 5606 + proxy_port: 5606 - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5605 + address: http://127.0.0.1:5606 upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) From f6ada9bfe3aaba8c4cc5849ae38a32ec5e30264d Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 2 Apr 2026 10:27:29 -0400 Subject: [PATCH 055/165] putting secure back in --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index b988e2f..e29c32e 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -453,7 +453,7 @@ instance_groups: release: secureproxy properties: secureproxy: - listen_port: 5606 + listen_port: 5605 proxy_port: 5606 - name: oauth2-proxy release: oauth2-proxy From e792605c8edd6891505569a63e428362f9652e3c Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 2 Apr 2026 12:29:30 -0400 Subject: [PATCH 056/165] putting lb in --- opensearch-base.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index e29c32e..335114b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -483,7 +483,8 @@ instance_groups: release: routing vm_extensions: - 15GB_ephemeral_disk - - logs-opensearch-profile + - logs-opensearch-profilev + - platform-opensearch-lb stemcell: default azs: [z1] networks: From d83d64093a10c2ccdbb6135e6fb464d206e0209a Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 2 Apr 2026 12:31:30 -0400 Subject: [PATCH 057/165] putting lb in --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 335114b..2488f2a 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -483,7 +483,7 @@ instance_groups: release: routing vm_extensions: - 15GB_ephemeral_disk - - logs-opensearch-profilev + - logs-opensearch-profile - platform-opensearch-lb stemcell: default azs: [z1] From 2c2ae60179d1386df3a700162b754ecdec5f99d0 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 2 Apr 2026 13:55:41 -0400 Subject: [PATCH 058/165] this should enable oidc --- opensearch-base.yml | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 2488f2a..e97e1db 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -97,9 +97,10 @@ instance_groups: opensearch: oidc: enabled: true - hostname: ((oidc_host)) client_id: ((oidc_client)) client_secret: ((oidc_secret)) + oidc_issuer_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) node: attributes: box_type: hot @@ -156,9 +157,10 @@ instance_groups: opensearch: oidc: enabled: true - hostname: ((oidc_host)) client_id: ((oidc_client)) client_secret: ((oidc_secret)) + oidc_issuer_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) clustername: opensearch limits: fd: 131072 # 2 ** 17 @@ -420,9 +422,10 @@ instance_groups: opensearch: oidc: enabled: true - hostname: ((oidc_host)) client_id: ((oidc_client)) client_secret: ((oidc_secret)) + oidc_issuer_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) heap_size: 1G http_host: 127.0.0.1 jvm_options: @@ -437,7 +440,7 @@ instance_groups: server: ssl: enabled: false - port: 5607 + port: 5605 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production @@ -449,24 +452,6 @@ instance_groups: tenants: enable_private: false release: opensearch - - name: secureproxy - release: secureproxy - properties: - secureproxy: - listen_port: 5605 - proxy_port: 5606 - - name: oauth2-proxy - release: oauth2-proxy - properties: - address: http://127.0.0.1:5606 - upstream: http://127.0.0.1:5607 - provider: oidc - client_id: ((client_id)) - client_secret: ((client_secret)) - cookie_secret: ((cookie_secret)) - oidc_issuer_url: ((oidc_issuer_url)) - redirect_url: ((redirect_url)) - email_domain: gsa.gov - name: route_registrar release: route-registrar consumes: From d411a9d493f5912f2424b79785eab02cd81a1b8a Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 2 Apr 2026 14:07:01 -0400 Subject: [PATCH 059/165] this should enable oidc --- opensearch-base.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index e97e1db..1f865fc 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -64,8 +64,8 @@ instance_groups: oidc: enabled: true hostname: ((oidc_host)) - client_id: ((oidc_client)) - client_secret: ((oidc_secret)) + client_id: ((client_id)) + client_secret: ((client_secret)) jvm_options: - "-Dlog4j2.formatMsgNoLookups=true" cf: @@ -97,8 +97,8 @@ instance_groups: opensearch: oidc: enabled: true - client_id: ((oidc_client)) - client_secret: ((oidc_secret)) + client_id: ((client_id)) + client_secret: ((client_secret)) oidc_issuer_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) node: @@ -157,8 +157,8 @@ instance_groups: opensearch: oidc: enabled: true - client_id: ((oidc_client)) - client_secret: ((oidc_secret)) + client_id: ((client_id)) + client_secret: ((client_secret)) oidc_issuer_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) clustername: opensearch @@ -206,8 +206,8 @@ instance_groups: oidc: enabled: true hostname: ((oidc_host)) - client_id: ((oidc_client)) - client_secret: ((oidc_secret)) + client_id: ((client_id)) + client_secret: ((client_secret)) clustername: opensearch limits: fd: 131072 # 2 ** 17 @@ -340,8 +340,8 @@ instance_groups: oidc: enabled: true hostname: ((oidc_host)) - client_id: ((oidc_client)) - client_secret: ((oidc_secret)) + client_id: ((client_id)) + client_secret: ((client_secret)) - name: ingestor_syslog consumes: *consumes-opensearch-manager properties: @@ -422,8 +422,8 @@ instance_groups: opensearch: oidc: enabled: true - client_id: ((oidc_client)) - client_secret: ((oidc_secret)) + client_id: ((client_id)) + client_secret: ((client_secret)) oidc_issuer_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) heap_size: 1G From 321abae9e154c41250f5dfbe99db8a4c785d6f54 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 3 Apr 2026 11:08:53 -0400 Subject: [PATCH 060/165] enabling post --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 1f865fc..08c5d51 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -110,7 +110,7 @@ instance_groups: fd: 131072 # 2 ** 17 health: timeout: 900 - disable_post_start: true + disable_post_start: false recovery: delay_allocation_restart: "10m" config_options: From cb785ba2980eb4589683e444809ced19d566c739 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 3 Apr 2026 12:00:58 -0400 Subject: [PATCH 061/165] updating oidc --- opensearch-base.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 08c5d51..4594c63 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -99,7 +99,7 @@ instance_groups: enabled: true client_id: ((client_id)) client_secret: ((client_secret)) - oidc_issuer_url: ((oidc_issuer_url)) + oidc_issuer_url: ((openid_connect_url)) redirect_url: ((redirect_url)) node: attributes: @@ -159,7 +159,7 @@ instance_groups: enabled: true client_id: ((client_id)) client_secret: ((client_secret)) - oidc_issuer_url: ((oidc_issuer_url)) + oidc_issuer_url: ((openid_connect_url)) redirect_url: ((redirect_url)) clustername: opensearch limits: @@ -424,7 +424,7 @@ instance_groups: enabled: true client_id: ((client_id)) client_secret: ((client_secret)) - oidc_issuer_url: ((oidc_issuer_url)) + oidc_issuer_url: ((openid_connect_url)) redirect_url: ((redirect_url)) heap_size: 1G http_host: 127.0.0.1 From 5ff69ec84a75e073d46c678175dd11cecd5d6e8b Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 3 Apr 2026 12:05:12 -0400 Subject: [PATCH 062/165] updating with correct stuff --- opensearch-base.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 4594c63..6d40368 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -99,7 +99,7 @@ instance_groups: enabled: true client_id: ((client_id)) client_secret: ((client_secret)) - oidc_issuer_url: ((openid_connect_url)) + openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) node: attributes: @@ -159,7 +159,7 @@ instance_groups: enabled: true client_id: ((client_id)) client_secret: ((client_secret)) - oidc_issuer_url: ((openid_connect_url)) + openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) clustername: opensearch limits: @@ -424,7 +424,7 @@ instance_groups: enabled: true client_id: ((client_id)) client_secret: ((client_secret)) - oidc_issuer_url: ((openid_connect_url)) + openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) heap_size: 1G http_host: 127.0.0.1 From 702165fea153f16fff3f244176329c8ea888d9f4 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 3 Apr 2026 14:31:03 -0400 Subject: [PATCH 063/165] the link for opensearhc.oidc is not working --- opensearch-base.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 6d40368..5bb0a2d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -63,9 +63,10 @@ instance_groups: indices.query.bool.max_clause_count: 2048 oidc: enabled: true - hostname: ((oidc_host)) client_id: ((client_id)) client_secret: ((client_secret)) + openid_connect_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) jvm_options: - "-Dlog4j2.formatMsgNoLookups=true" cf: From f0a51a2fcc81f93b228fa4c5fb4e32f35518b84c Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 3 Apr 2026 15:23:11 -0400 Subject: [PATCH 064/165] adding in redirect --- opensearch-base.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 5bb0a2d..fabaed5 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -435,6 +435,8 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch_dashboards: + oidc: + redirect: "logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov" config_options: server.maxPayloadBytes: 4194304 console.enabled: false From 7f0422a792cb9526cdedb9af1d06345a0add2d55 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 11:23:25 -0400 Subject: [PATCH 065/165] removing test for testing --- ci/pipeline.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 7e7ff3f..66a4f2b 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -18,21 +18,6 @@ jobs: - get: releases-dir-tarball resource: opensearch-releases-dir-tarball - get: general-task - - task: run-tests - image: general-task - config: - platform: linux - inputs: - - name: release-git-repo - run: - path: sh - args: - - -exc - - | - cd release-git-repo - bundle install - # run release template unit tests - bundle exec rspec - task: create-test-release input_mapping: deploy-logs-opensearch-config: deploy-logs-platform-config From 9424a653ffdea314164eebfbd4d53661589e45cf Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 14:57:30 -0400 Subject: [PATCH 066/165] getting this to work --- opensearch-base.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index fabaed5..2369bf5 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -443,7 +443,7 @@ instance_groups: server: ssl: enabled: false - port: 5605 + port: 5606 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production @@ -455,6 +455,19 @@ instance_groups: tenants: enable_private: false release: opensearch + - name: oauth2-proxy + release: oauth2-proxy + properties: + address: http://0.0.0.0:8085 + upstream: http://127.0.0.1:8086 + provider: oidc + client_id: ((oauth-proxy-client-id)) + client_secret: ((oauth-proxy-client-secret)) + cookie_secret: ((oauth-proxy-cookie-secret)) + oidc_issuer_url: ((oauth-proxy-oidc-issuer-url)) + email_domain: gsa.gov + browser_xss_filter: True + content_type_nosniff: True - name: route_registrar release: route-registrar consumes: From dea77d567177e71ab4d3cd89d9fddab85e63db67 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 15:10:22 -0400 Subject: [PATCH 067/165] getting this to work --- opensearch-base.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 2369bf5..905ad19 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -461,10 +461,10 @@ instance_groups: address: http://0.0.0.0:8085 upstream: http://127.0.0.1:8086 provider: oidc - client_id: ((oauth-proxy-client-id)) - client_secret: ((oauth-proxy-client-secret)) - cookie_secret: ((oauth-proxy-cookie-secret)) - oidc_issuer_url: ((oauth-proxy-oidc-issuer-url)) + client_id: ((client-id)) + client_secret: ((client-secret)) + # cookie_secret: ((oauth-proxy-cookie-secret)) + oidc_issuer_url: ((oidc-issuer-url)) email_domain: gsa.gov browser_xss_filter: True content_type_nosniff: True From d0dc41af03fd2b109770da3beb1420ab53c7d833 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 15:14:14 -0400 Subject: [PATCH 068/165] getting this to work --- opensearch-base.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 905ad19..56bb1f2 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -461,10 +461,10 @@ instance_groups: address: http://0.0.0.0:8085 upstream: http://127.0.0.1:8086 provider: oidc - client_id: ((client-id)) - client_secret: ((client-secret)) + client_id: ((client_id)) + client_secret: ((client_secret)) # cookie_secret: ((oauth-proxy-cookie-secret)) - oidc_issuer_url: ((oidc-issuer-url)) + oidc_issuer_url: ((oidc_issuer_url)) email_domain: gsa.gov browser_xss_filter: True content_type_nosniff: True From ee6192a4d343cb80028581d342e90b95d90ca876 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 15:19:02 -0400 Subject: [PATCH 069/165] need cookie --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 56bb1f2..00c182e 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -463,7 +463,7 @@ instance_groups: provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) - # cookie_secret: ((oauth-proxy-cookie-secret)) + cookie_secret: ((cookie-secret)) oidc_issuer_url: ((oidc_issuer_url)) email_domain: gsa.gov browser_xss_filter: True From 29ef6dd4011cd29978f0a6a887a3c3a246bd214d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 15:20:59 -0400 Subject: [PATCH 070/165] need cookie --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 00c182e..29eecc5 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -463,7 +463,7 @@ instance_groups: provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) - cookie_secret: ((cookie-secret)) + cookie_secret: ((cookie_secret)) oidc_issuer_url: ((oidc_issuer_url)) email_domain: gsa.gov browser_xss_filter: True From e53144e739f3e239d6c5a26752c6f0592564c62d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 16:08:45 -0400 Subject: [PATCH 071/165] fixing port --- opensearch-base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 29eecc5..0f1830b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -458,8 +458,8 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://0.0.0.0:8085 - upstream: http://127.0.0.1:8086 + address: http://0.0.0.0:5605 + upstream: http://127.0.0.1:5606 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) From 7a4e026ce4dc638fde7ead7eae85b3cc06b1de62 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 16:09:54 -0400 Subject: [PATCH 072/165] adding in secureproxy --- opensearch-base.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 0f1830b..e522db0 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -443,7 +443,7 @@ instance_groups: server: ssl: enabled: false - port: 5606 + port: 5607 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production @@ -458,8 +458,8 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://0.0.0.0:5605 - upstream: http://127.0.0.1:5606 + address: http://0.0.0.0:5606 + upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) @@ -468,6 +468,12 @@ instance_groups: email_domain: gsa.gov browser_xss_filter: True content_type_nosniff: True + - name: secureproxy + properties: + secureproxy: + listen_port: 5605 + proxy_port: 5606 + release: secureproxy - name: route_registrar release: route-registrar consumes: From 3a591449b236c5055cd69632a18ec5ae015980e1 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 16:52:47 -0400 Subject: [PATCH 073/165] adding in secureproxy --- opensearch-base.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index e522db0..e34cd62 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -466,8 +466,6 @@ instance_groups: cookie_secret: ((cookie_secret)) oidc_issuer_url: ((oidc_issuer_url)) email_domain: gsa.gov - browser_xss_filter: True - content_type_nosniff: True - name: secureproxy properties: secureproxy: From cdfc811ec26da68127d9006f115badd74b063a6a Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 17:06:24 -0400 Subject: [PATCH 074/165] adding in secureproxy --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index e34cd62..7598e96 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -459,7 +459,7 @@ instance_groups: release: oauth2-proxy properties: address: http://0.0.0.0:5606 - upstream: http://127.0.0.1:5607 + upstream: http://0.0.0.0:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) From b8f19823c8edb9929e6bfdc65b057b2a41f95f76 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 17:49:29 -0400 Subject: [PATCH 075/165] adding in secureproxy --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 7598e96..2936b9b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -465,6 +465,7 @@ instance_groups: client_secret: ((client_secret)) cookie_secret: ((cookie_secret)) oidc_issuer_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) email_domain: gsa.gov - name: secureproxy properties: From d39bd8018be65fa15bf2e593de7a67cbd47b321a Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 17:56:56 -0400 Subject: [PATCH 076/165] adding in secureproxy --- opensearch-base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 2936b9b..6d84258 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -458,8 +458,8 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://0.0.0.0:5606 - upstream: http://0.0.0.0:5607 + address: http://127.0.0.1:5606 + upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) From 676a50d618156dbd1330f2de86652f638a5a43a7 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 18:06:58 -0400 Subject: [PATCH 077/165] adding in secureproxy --- opensearch-base.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 6d84258..022f00c 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -455,23 +455,23 @@ instance_groups: tenants: enable_private: false release: opensearch - - name: oauth2-proxy - release: oauth2-proxy - properties: - address: http://127.0.0.1:5606 - upstream: http://127.0.0.1:5607 - provider: oidc - client_id: ((client_id)) - client_secret: ((client_secret)) - cookie_secret: ((cookie_secret)) - oidc_issuer_url: ((oidc_issuer_url)) - redirect_url: ((redirect_url)) - email_domain: gsa.gov + # - name: oauth2-proxy + # release: oauth2-proxy + # properties: + # address: http://127.0.0.1:5606 + # upstream: http://127.0.0.1:5607 + # provider: oidc + # client_id: ((client_id)) + # client_secret: ((client_secret)) + # cookie_secret: ((cookie_secret)) + # oidc_issuer_url: ((oidc_issuer_url)) + # redirect_url: ((redirect_url)) + # email_domain: gsa.gov - name: secureproxy properties: secureproxy: listen_port: 5605 - proxy_port: 5606 + proxy_port: 5607 release: secureproxy - name: route_registrar release: route-registrar From 497f5102e0b2a5e2af40dfcc245d90884b91ba15 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 18:21:00 -0400 Subject: [PATCH 078/165] fixed credhub values --- opensearch-base.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 022f00c..35567dd 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -455,18 +455,18 @@ instance_groups: tenants: enable_private: false release: opensearch - # - name: oauth2-proxy - # release: oauth2-proxy - # properties: - # address: http://127.0.0.1:5606 - # upstream: http://127.0.0.1:5607 - # provider: oidc - # client_id: ((client_id)) - # client_secret: ((client_secret)) - # cookie_secret: ((cookie_secret)) - # oidc_issuer_url: ((oidc_issuer_url)) - # redirect_url: ((redirect_url)) - # email_domain: gsa.gov + - name: oauth2-proxy + release: oauth2-proxy + properties: + address: http://127.0.0.1:5606 + upstream: http://127.0.0.1:5607 + provider: oidc + client_id: ((client_id)) + client_secret: ((client_secret)) + cookie_secret: ((cookie_secret)) + oidc_issuer_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) + email_domain: gsa.gov - name: secureproxy properties: secureproxy: From 90711886cdb211c36b20f6554c49f94448761307 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 19:37:23 -0400 Subject: [PATCH 079/165] port mishap --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 35567dd..6d84258 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -471,7 +471,7 @@ instance_groups: properties: secureproxy: listen_port: 5605 - proxy_port: 5607 + proxy_port: 5606 release: secureproxy - name: route_registrar release: route-registrar From 872e613c5be3859cc439c7194be4af3e5be7e95d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 19:56:32 -0400 Subject: [PATCH 080/165] port mishap --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 6d84258..5c43d83 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -459,7 +459,7 @@ instance_groups: release: oauth2-proxy properties: address: http://127.0.0.1:5606 - upstream: http://127.0.0.1:5607 + upstream: https://127.0.0.1:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) From 1258c1cb293a2b4eca1f8dfc76f12a0a47c17f79 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 6 Apr 2026 20:16:52 -0400 Subject: [PATCH 081/165] removing --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 5c43d83..6d84258 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -459,7 +459,7 @@ instance_groups: release: oauth2-proxy properties: address: http://127.0.0.1:5606 - upstream: https://127.0.0.1:5607 + upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) client_secret: ((client_secret)) From e4b026d2ba81dfca3821b3f6f07660959814d89f Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 09:52:30 -0400 Subject: [PATCH 082/165] removing secure-proxy --- ci/pipeline.yml | 2 +- opensearch-base.yml | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 66a4f2b..f0aaada 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -93,7 +93,7 @@ jobs: ops_files: - deploy-logs-platform-config/opsfiles/enable-node-tls.yml - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + #- deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml diff --git a/opensearch-base.yml b/opensearch-base.yml index 6d84258..e20a8b3 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -458,7 +458,7 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5606 + address: http://127.0.0.1:5605 upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) @@ -467,12 +467,6 @@ instance_groups: oidc_issuer_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) email_domain: gsa.gov - - name: secureproxy - properties: - secureproxy: - listen_port: 5605 - proxy_port: 5606 - release: secureproxy - name: route_registrar release: route-registrar consumes: From d38fbf6154d6b1930ea8c19cf46808871a33e16a Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 11:25:33 -0400 Subject: [PATCH 083/165] longer interval better name --- ci/pipeline.yml | 2 +- opensearch-base.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index f0aaada..66a4f2b 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -93,7 +93,7 @@ jobs: ops_files: - deploy-logs-platform-config/opsfiles/enable-node-tls.yml - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - #- deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml diff --git a/opensearch-base.yml b/opensearch-base.yml index e20a8b3..426049e 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -475,9 +475,9 @@ instance_groups: properties: route_registrar: routes: - - name: platform-auth-proxy + - name: platform-uri port: 5605 - registration_interval: 2s + registration_interval: 10s uris: - logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov release: routing From ece711b4f5c9654cd9a85569e0c630ca7c3eadf1 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 11:35:54 -0400 Subject: [PATCH 084/165] maybe this is it --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 426049e..0d18024 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -458,7 +458,7 @@ instance_groups: - name: oauth2-proxy release: oauth2-proxy properties: - address: http://127.0.0.1:5605 + address: 0.0.0.0:5605 upstream: http://127.0.0.1:5607 provider: oidc client_id: ((client_id)) From b872ee977dd8aa5662eb8d1f38350bab501a1c4a Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 11:41:23 -0400 Subject: [PATCH 085/165] maybe this is it --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 0d18024..f072c19 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -449,6 +449,7 @@ instance_groups: - NODE_ENV: production health: timeout: 600 + disable_post_start: true index: ((dashboard_index)) memory_limit: 75 multitenancy: From 20f7ba7604aa8b33bf90f3f581704ef0916395f3 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 12:30:55 -0400 Subject: [PATCH 086/165] maybe this is it --- ci/pipeline.yml | 2 +- opensearch-base.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 66a4f2b..3e51f1d 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -93,7 +93,7 @@ jobs: ops_files: - deploy-logs-platform-config/opsfiles/enable-node-tls.yml - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + # - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml diff --git a/opensearch-base.yml b/opensearch-base.yml index f072c19..6f9656f 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -465,7 +465,7 @@ instance_groups: client_id: ((client_id)) client_secret: ((client_secret)) cookie_secret: ((cookie_secret)) - oidc_issuer_url: ((oidc_issuer_url)) + oidc_issuer_url: ((oauth2_oidc_issuer_url)) redirect_url: ((redirect_url)) email_domain: gsa.gov - name: route_registrar From b9558d7ba011166e3ade665ed9b5895b1685b69a Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 13:17:54 -0400 Subject: [PATCH 087/165] direct oidc --- opensearch-base.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 6f9656f..27ba4b6 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -443,7 +443,7 @@ instance_groups: server: ssl: enabled: false - port: 5607 + port: 5605 defaultAppId: dashboard/App-Overview env: - NODE_ENV: production @@ -456,18 +456,18 @@ instance_groups: tenants: enable_private: false release: opensearch - - name: oauth2-proxy - release: oauth2-proxy - properties: - address: 0.0.0.0:5605 - upstream: http://127.0.0.1:5607 - provider: oidc - client_id: ((client_id)) - client_secret: ((client_secret)) - cookie_secret: ((cookie_secret)) - oidc_issuer_url: ((oauth2_oidc_issuer_url)) - redirect_url: ((redirect_url)) - email_domain: gsa.gov + # - name: oauth2-proxy + # release: oauth2-proxy + # properties: + # address: 0.0.0.0:5605 + # upstream: http://127.0.0.1:5607 + # provider: oidc + # client_id: ((client_id)) + # client_secret: ((client_secret)) + # cookie_secret: ((cookie_secret)) + # oidc_issuer_url: ((oauth2_oidc_issuer_url)) + # redirect_url: ((redirect_url)) + # email_domain: gsa.gov - name: route_registrar release: route-registrar consumes: From fd6b13fb56aa577ae8bd5dd3e47a71c8e9a768aa Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 16:23:18 -0400 Subject: [PATCH 088/165] updating tenant --- opensearch-base.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 27ba4b6..d34dfa7 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -454,20 +454,8 @@ instance_groups: memory_limit: 75 multitenancy: tenants: - enable_private: false + enabled: false release: opensearch - # - name: oauth2-proxy - # release: oauth2-proxy - # properties: - # address: 0.0.0.0:5605 - # upstream: http://127.0.0.1:5607 - # provider: oidc - # client_id: ((client_id)) - # client_secret: ((client_secret)) - # cookie_secret: ((cookie_secret)) - # oidc_issuer_url: ((oauth2_oidc_issuer_url)) - # redirect_url: ((redirect_url)) - # email_domain: gsa.gov - name: route_registrar release: route-registrar consumes: From 835f63d087045c687c7e793b0b467e33cc25bb52 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 7 Apr 2026 16:45:14 -0400 Subject: [PATCH 089/165] disabled tenancy --- opensearch-base.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index d34dfa7..e2b9ede 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -453,8 +453,7 @@ instance_groups: index: ((dashboard_index)) memory_limit: 75 multitenancy: - tenants: - enabled: false + enabled: false release: opensearch - name: route_registrar release: route-registrar From 53edc6bbf5246344d7b4d9f286fc91fff2fba45a Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 8 Apr 2026 13:04:47 -0400 Subject: [PATCH 090/165] this will enable ssl --- ci/pipeline.yml | 3 ++- opsfiles/enable-oidc-tls.yml | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 opsfiles/enable-oidc-tls.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 3e51f1d..8cf89ad 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -93,7 +93,8 @@ jobs: ops_files: - deploy-logs-platform-config/opsfiles/enable-node-tls.yml - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - # - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml diff --git a/opsfiles/enable-oidc-tls.yml b/opsfiles/enable-oidc-tls.yml new file mode 100644 index 0000000..692d529 --- /dev/null +++ b/opsfiles/enable-oidc-tls.yml @@ -0,0 +1,6 @@ +- type: replace + path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties/opensearch_dashboards/oidc?/ssl? + value: + ca: ((lets_encrypt_ca.certificate)) + certificate: ((opsuaa_uaa_ssl.certificate)) + private_key: ((opsuaa_uaa_ssl.private_key)) \ No newline at end of file From 8f3fa74577ecbb266aa5c49953103e56ca291520 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 8 Apr 2026 13:39:01 -0400 Subject: [PATCH 091/165] making it be found --- opsfiles/enable-oidc-tls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opsfiles/enable-oidc-tls.yml b/opsfiles/enable-oidc-tls.yml index 692d529..ee95452 100644 --- a/opsfiles/enable-oidc-tls.yml +++ b/opsfiles/enable-oidc-tls.yml @@ -1,6 +1,6 @@ - type: replace path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties/opensearch_dashboards/oidc?/ssl? value: - ca: ((lets_encrypt_ca.certificate)) + ca: ((prometheus_production_lets_encrypt_ca.certificate)) certificate: ((opsuaa_uaa_ssl.certificate)) private_key: ((opsuaa_uaa_ssl.private_key)) \ No newline at end of file From 95c40933dd65d930c5f310f87e1bc11dae002fd9 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 8 Apr 2026 14:11:02 -0400 Subject: [PATCH 092/165] making it be found --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index e2b9ede..cdb534d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -422,7 +422,6 @@ instance_groups: properties: opensearch: oidc: - enabled: true client_id: ((client_id)) client_secret: ((client_secret)) openid_connect_url: ((oidc_issuer_url)) @@ -436,6 +435,7 @@ instance_groups: properties: opensearch_dashboards: oidc: + enabled: true redirect: "logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov" config_options: server.maxPayloadBytes: 4194304 From 480173c67cf311c5aaf1d0be70cfe0499b627dc4 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 9 Apr 2026 13:17:09 -0400 Subject: [PATCH 093/165] no route registrar --- opensearch-base.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index cdb534d..8ed561b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -455,20 +455,6 @@ instance_groups: multitenancy: enabled: false release: opensearch - - name: route_registrar - release: route-registrar - consumes: - nats-tls: - from: nats-tls - properties: - route_registrar: - routes: - - name: platform-uri - port: 5605 - registration_interval: 10s - uris: - - logs-platform-test.dev.us-gov-west-1.aws-us-gov.cloud.gov - release: routing vm_extensions: - 15GB_ephemeral_disk - logs-opensearch-profile From d9b9519996618b46b271f3e2f00643e067db2b9a Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 9 Apr 2026 13:34:52 -0400 Subject: [PATCH 094/165] remove route regsitrar --- ci/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 8cf89ad..a996251 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -96,7 +96,7 @@ jobs: - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml + # - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml - deploy-logs-platform-config/opsfiles/cf-development.yml - deploy-logs-platform-config/opsfiles/opensearch-notification.yml From 549af52d22d54ee2a371509f63ee430cc114651c Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 10 Apr 2026 09:56:59 -0400 Subject: [PATCH 095/165] first attempt at platform opensearch ingestor --- opensearch-base.yml | 71 +++++++++++++++++------------- opensearch-scaling-development.yml | 2 +- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 8ed561b..9778a15 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -346,38 +346,12 @@ instance_groups: - name: ingestor_syslog consumes: *consumes-opensearch-manager properties: - outputs: - - plugin: s3 - options: - region: ((region)) - bucket: ((bucket)) - access_key_id: ((access_key_id)) - secret_access_key: ((secret_access_key)) - server_side_encryption: true - prefix: "%{+yyyy/MM/dd/HH/mm}" - encoding: "gzip" - temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp - # note that this is different than in the archivers - # here, we're uploading the parsed message as json, not the raw log line - codec: json - - plugin: opensearch - options: {} logstash: jvm_options: - -Dlog4j2.formatMsgNoLookups=true queue: max_bytes: 30gb - logstash_ingestor: - syslog_tls: - port: 6972 - ssl_cert: ((ingestor_syslog_server_tls.certificate)) - ssl_key: ((ingestor_syslog_server_tls.private_key)) logstash_parser: - deployment_dictionary: - - /var/vcap/packages/base-logstash-filters/deployment_lookup.yml - - /var/vcap/jobs/deployment_lookup_config/config/deployment_lookup.yml - filters: - - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-filters-default.conf opensearch: data_hosts: - localhost @@ -387,19 +361,56 @@ instance_groups: ca: ((opensearch_node.ca)) certificate: ((logstash.certificate)) private_key: ((logstash.private_key)) + filters: + - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-filters-default.conf + - content: | + if [@source][component] == "snort" { + grok { + match => { + "@message" => "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:msg}\s\{%{DATA:proto}\}\s%{IP:src_ip}:%{INT:src_port}\s->\s%{IP:dst_ip}:%{INT:dst_port}" + } + } + } + if [@source][component] == "clamd" { + grok { + match => { + "@message" => "%{WORD:event_type}:\s%{DATA:file_path}:\s%{DATA:signature_name}\sFOUND" + } + } + } + outputs: + # - plugin: s3 + # options: + # region: ((region)) + # bucket: ((bucket)) + # server_side_encryption: true + # prefix: "%{+yyyy/MM/dd/HH/mm}" + # encoding: "gzip" + # temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp + # # note that this is different than in the archivers + # # here, we're uploading the parsed message as json, not the raw log line + # codec: json + - plugin: opensearch + options: {} + deployment_dictionary: + - /var/vcap/packages/base-logstash-filters/deployment_lookup.yml + - /var/vcap/jobs/deployment_lookup_config/config/deployment_lookup.yml + logstash_ingestor: + syslog: + port: 5514 provides: ingestor: as: ingestor_link release: opensearch - name: deployment_lookup_config release: opensearch - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - azs: [z1,z2] vm_type: t3.medium vm_extensions: - - logs-opensearch-ingestor-profile + - platform-syslog-lb - 20GB_ephemeral_disk + persistent_disk_type: logs_opensearch_ingestor + stemcell: default + azs: [z1,z2] networks: - name: services update: diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index bb874c1..cd2bd26 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -57,7 +57,7 @@ - type: replace path: /instance_groups/name=ingestor?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=ingestor?/vm_type? From 1313c14a2530572294a79120275c69b085d4285d Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 10 Apr 2026 09:59:23 -0400 Subject: [PATCH 096/165] first attempt at platform opensearch ingestor --- opensearch-base.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 9778a15..658f3a2 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -340,9 +340,10 @@ instance_groups: - -Dlog4j2.formatMsgNoLookups=true oidc: enabled: true - hostname: ((oidc_host)) client_id: ((client_id)) client_secret: ((client_secret)) + openid_connect_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) - name: ingestor_syslog consumes: *consumes-opensearch-manager properties: From ae370a7045a33323f97209068180e3caba302625 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 14 Apr 2026 11:14:00 -0400 Subject: [PATCH 097/165] 5431 instead --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 658f3a2..671a59f 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -398,7 +398,7 @@ instance_groups: - /var/vcap/jobs/deployment_lookup_config/config/deployment_lookup.yml logstash_ingestor: syslog: - port: 5514 + port: 5431 provides: ingestor: as: ingestor_link From ede0c4619af6b44c8292303588cdd1ea22ffedb5 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 14 Apr 2026 16:33:51 -0400 Subject: [PATCH 098/165] changing input --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 671a59f..18a6a04 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -363,7 +363,7 @@ instance_groups: certificate: ((logstash.certificate)) private_key: ((logstash.private_key)) filters: - - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-filters-default.conf + - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-platform-default.conf - content: | if [@source][component] == "snort" { grok { From e2670a744c9eae90c7953cd173985d950ed9ffc7 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 15 Apr 2026 08:21:09 -0400 Subject: [PATCH 099/165] setting to platform type --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 18a6a04..602a268 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -397,6 +397,7 @@ instance_groups: - /var/vcap/packages/base-logstash-filters/deployment_lookup.yml - /var/vcap/jobs/deployment_lookup_config/config/deployment_lookup.yml logstash_ingestor: + type: platform syslog: port: 5431 provides: From e76196b7b18b88038ef59f52b99816bfe262b75f Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 15 Apr 2026 10:08:54 -0400 Subject: [PATCH 100/165] switch to bosh dns --- ci/pipeline.yml | 1 + opsfiles/enable-ingestor-dns.yml | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 opsfiles/enable-ingestor-dns.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml index a996251..497b825 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -95,6 +95,7 @@ jobs: - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml + - deploy-logs-platform-config/opsfiles/enable-ingestor-dns.yml # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml # - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml diff --git a/opsfiles/enable-ingestor-dns.yml b/opsfiles/enable-ingestor-dns.yml new file mode 100644 index 0000000..3111abe --- /dev/null +++ b/opsfiles/enable-ingestor-dns.yml @@ -0,0 +1,10 @@ +- type: replace + path: /addons/name=bosh-dns-aliases/jobs/name=bosh-dns-aliases/properties?/aliases?/- + value: + domain: ingestor.platform_opensearch.internal + targets: + - query: '*' + instance_group: ingestor + deployment: logs-platform + network: services + domain: bosh \ No newline at end of file From 5d9929da65d15c1821025904f3d11a5fa0f811e6 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 15 Apr 2026 10:56:51 -0400 Subject: [PATCH 101/165] adding a debug option to see what log is erroring --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 602a268..35da89b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -353,6 +353,7 @@ instance_groups: queue: max_bytes: 30gb logstash_parser: + debug: true opensearch: data_hosts: - localhost From 0620975ea848a441d5f223a7d1c40b38a316b002 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 15 Apr 2026 11:43:44 -0400 Subject: [PATCH 102/165] turning off debug --- opensearch-base.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 35da89b..602a268 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -353,7 +353,6 @@ instance_groups: queue: max_bytes: 30gb logstash_parser: - debug: true opensearch: data_hosts: - localhost From 675e8e88905baa88e028a4578a049363490a3937 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 16 Apr 2026 09:43:57 -0400 Subject: [PATCH 103/165] better logging for these clamv --- opensearch-base.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 602a268..9c68e54 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -364,21 +364,6 @@ instance_groups: private_key: ((logstash.private_key)) filters: - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-platform-default.conf - - content: | - if [@source][component] == "snort" { - grok { - match => { - "@message" => "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:msg}\s\{%{DATA:proto}\}\s%{IP:src_ip}:%{INT:src_port}\s->\s%{IP:dst_ip}:%{INT:dst_port}" - } - } - } - if [@source][component] == "clamd" { - grok { - match => { - "@message" => "%{WORD:event_type}:\s%{DATA:file_path}:\s%{DATA:signature_name}\sFOUND" - } - } - } outputs: # - plugin: s3 # options: From 57e6fd8af4d1a661721386d18181914f0e59eabf Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 16 Apr 2026 13:07:55 -0400 Subject: [PATCH 104/165] adding in platform specific variables and removal of tenants --- ci/pipeline.yml | 1 + opensearch-base.yml | 28 +++++++++++++++------------- opsfiles/cf-development.yml | 7 ------- opsfiles/cf-production.yml | 6 ------ opsfiles/cf-staging.yml | 6 ------ 5 files changed, 16 insertions(+), 32 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 497b825..062c682 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -84,6 +84,7 @@ jobs: file: deploy-logs-platform-config/ci/terraform-secrets.yml - put: opensearch-development-deployment params: + dry_run: true manifest: opensearch-manifest/manifest.yml # dry_run: true releases: diff --git a/opensearch-base.yml b/opensearch-base.yml index 9c68e54..3218c25 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -47,6 +47,8 @@ instance_groups: ip_addresses: true properties: opensearch: + deployment_type: + - platform node: attributes: box_type: warm @@ -96,6 +98,8 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + deployment_type: + - platform oidc: enabled: true client_id: ((client_id)) @@ -156,6 +160,8 @@ instance_groups: as: opensearch_manager properties: opensearch: + deployment_type: + - platform oidc: enabled: true client_id: ((client_id)) @@ -204,6 +210,8 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + deployment_type: + - platform oidc: enabled: true hostname: ((oidc_host)) @@ -248,11 +256,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch_config: - alias: ((alias)) - metric_alias: ((metric_alias)) + platform_alias: ((platform_alias)) component_templates: - - shards-and-replicas: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas.json - - metric-shards-and-replica: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-metrics.json + - platform-shards-and-replica: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json - index-settings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-settings.json - index-mappings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-mappings.json - component-index-mappings-base: /var/vcap/jobs/opensearch_templates/component-index-mappings-base.json @@ -263,13 +269,6 @@ instance_groups: - index-template-app-cf: /var/vcap/jobs/opensearch_templates/index-template-app.json - index-template-metrics: /var/vcap/jobs/opensearch_templates/index-template-metrics.json release: opensearch - - name: upload_tenant - consumes: *consumes-opensearch-manager - properties: - upload_tenant: - cf: - client_id: opensearch_client_id - release: opensearch - name: opensearch_templates properties: opensearch_config: @@ -285,8 +284,7 @@ instance_groups: index_settings_component_name: index-settings platform_index_component_name: index-mappings-platform metric_index_component_name: component-index-mappings-metrics - shards_and_replicas_component_name: shards-and-replicas - metrics_shards_and_replicas_component_name: metric-shards-and-replica + platform_shards_and_replicas_component_name: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json release: opensearch - name: upload-dashboards-objects release: opensearch @@ -334,6 +332,8 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + deployment_type: + - platform heap_size: 1G http_host: 127.0.0.1 jvm_options: @@ -419,6 +419,8 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + deployment_type: + - platform oidc: client_id: ((client_id)) client_secret: ((client_secret)) diff --git a/opsfiles/cf-development.yml b/opsfiles/cf-development.yml index d3ba390..9a9b6e1 100644 --- a/opsfiles/cf-development.yml +++ b/opsfiles/cf-development.yml @@ -1,11 +1,4 @@ # replace for client_password -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload_tenant/properties?/upload_tenant?/cf?/client_password? - value: ((/bosh/cf-development/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload_tenant/properties?/upload_tenant?/cf?/domain? - value: ((cf-api-development)) - - type: replace path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/client_password? value: ((/bosh/cf-development/opensearch_client_secret)) diff --git a/opsfiles/cf-production.yml b/opsfiles/cf-production.yml index 70f9694..17cce14 100644 --- a/opsfiles/cf-production.yml +++ b/opsfiles/cf-production.yml @@ -1,10 +1,4 @@ # replace for client_password -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload_tenant/properties?/upload_tenant?/cf?/client_password? - value: ((/bosh/cf-production/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload_tenant/properties?/upload_tenant?/cf?/domain? - value: ((cf-api-production)) - type: replace path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/client_password? diff --git a/opsfiles/cf-staging.yml b/opsfiles/cf-staging.yml index 156d9bb..0e6d25d 100644 --- a/opsfiles/cf-staging.yml +++ b/opsfiles/cf-staging.yml @@ -1,10 +1,4 @@ # replace for client_password -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload_tenant/properties?/upload_tenant?/cf?/client_password? - value: ((/bosh/cf-staging/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload_tenant/properties?/upload_tenant?/cf?/domain? - value: ((cf-api-staging)) - type: replace path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/client_password? From 17c758352015ed80e272f61c98342ce6345b3318 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 16 Apr 2026 16:22:44 -0400 Subject: [PATCH 105/165] one maintenance node --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index cd2bd26..b43da45 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -96,7 +96,7 @@ - type: replace path: /instance_groups/name=maintenance?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=maintenance?/vm_type? From ee54d58633aa0c437b3784eefa09d67245b3034f Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 17 Apr 2026 08:06:35 -0400 Subject: [PATCH 106/165] oidc fix --- opensearch-base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 3218c25..d447b05 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -213,10 +213,10 @@ instance_groups: deployment_type: - platform oidc: - enabled: true - hostname: ((oidc_host)) client_id: ((client_id)) client_secret: ((client_secret)) + openid_connect_url: ((oidc_issuer_url)) + redirect_url: ((redirect_url)) clustername: opensearch limits: fd: 131072 # 2 ** 17 From b4b42c5bbf18d89b309736839eb184a17a7857bd Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 17 Apr 2026 08:34:45 -0400 Subject: [PATCH 107/165] needs tls --- opensearch-base.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index d447b05..42fe050 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -385,6 +385,10 @@ instance_groups: type: platform syslog: port: 5431 + syslog_tls: + port: 6972 + ssl_cert: ((ingestor_syslog_server_tls.certificate)) + ssl_key: ((ingestor_syslog_server_tls.private_key)) provides: ingestor: as: ingestor_link From 9c76f03009d204d55b873eb3e27333621c01f621 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 17 Apr 2026 16:01:57 -0400 Subject: [PATCH 108/165] correct endpoint --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 42fe050..417e187 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -309,7 +309,7 @@ instance_groups: - {type: visualization, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/visualization/App-*.json"} - {type: dashboard, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/dashboard/App-*.json"} opensearch_dashboards: - host: opensearch_dashboards.platform-opensearch.internal + host: opensearch_dashboards.opensearch.internal stemcell: default azs: [z1,z2] vm_type: t3.large From ae7c9433557e6a68cea5590c27f09fbbc28e9a63 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 20 Apr 2026 09:56:20 -0400 Subject: [PATCH 109/165] token url addition --- opensearch-base.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 417e187..129e46f 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -69,6 +69,7 @@ instance_groups: client_secret: ((client_secret)) openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) + token_url: ((token_url)) jvm_options: - "-Dlog4j2.formatMsgNoLookups=true" cf: @@ -106,6 +107,7 @@ instance_groups: client_secret: ((client_secret)) openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) + token_url: ((token_url)) node: attributes: box_type: hot @@ -168,6 +170,7 @@ instance_groups: client_secret: ((client_secret)) openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) + token_url: ((token_url)) clustername: opensearch limits: fd: 131072 # 2 ** 17 @@ -217,6 +220,7 @@ instance_groups: client_secret: ((client_secret)) openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) + token_url: ((token_url)) clustername: opensearch limits: fd: 131072 # 2 ** 17 @@ -344,6 +348,7 @@ instance_groups: client_secret: ((client_secret)) openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) + token_url: ((token_url)) - name: ingestor_syslog consumes: *consumes-opensearch-manager properties: @@ -430,6 +435,7 @@ instance_groups: client_secret: ((client_secret)) openid_connect_url: ((oidc_issuer_url)) redirect_url: ((redirect_url)) + token_url: ((token_url)) heap_size: 1G http_host: 127.0.0.1 jvm_options: From 51056f412ef85ea7186347151c781a8ef611cb59 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 21 Apr 2026 10:43:32 -0400 Subject: [PATCH 110/165] updated to use platform components --- opensearch-base.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 129e46f..a6d1980 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -266,12 +266,10 @@ instance_groups: - index-settings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-settings.json - index-mappings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-mappings.json - component-index-mappings-base: /var/vcap/jobs/opensearch_templates/component-index-mappings-base.json - - component-index-mappings-app-cf: /var/vcap/jobs/opensearch_templates/component-index-mappings-app.json - - component-index-mappings-metrics: /var/vcap/jobs/opensearch_templates/component-index-mappings-metrics.json + - component-index-mappings-platform: /var/vcap/jobs/opensearch_templates/component-index-mappings-platform.json index_templates: - index-template: /var/vcap/jobs/opensearch_templates/index-template.json - - index-template-app-cf: /var/vcap/jobs/opensearch_templates/index-template-app.json - - index-template-metrics: /var/vcap/jobs/opensearch_templates/index-template-metrics.json + - index-templat-platform: /var/vcap/jobs/opensearch_templates/index-template-platform.json release: opensearch - name: opensearch_templates properties: From 3f50f3b40eed17ef576661bba20d88770256cd29 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 21 Apr 2026 11:12:40 -0400 Subject: [PATCH 111/165] changing to use platform instead --- opensearch-base.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index a6d1980..ee40568 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -235,14 +235,16 @@ instance_groups: name: smoke_tests properties: smoke_tests: + app_log: + enabled: false count_test: - index_pattern: logs-app-* + index_pattern: logs-platform-* minimum: 10 long_time_interval: 10m run: true time_field: '@timestamp' time_interval: 5m - index: logs-app + index: logs-platform opensearch: ssl: ca: ((opensearch_node.ca)) From bbd5ba1221e3b0944c6189636cc3654cf52dab99 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 21 Apr 2026 11:14:15 -0400 Subject: [PATCH 112/165] enable platform smoketest --- opensearch-base.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index ee40568..9e641b9 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -250,6 +250,8 @@ instance_groups: ca: ((opensearch_node.ca)) certificate: ((smoketest.certificate)) private_key: ((smoketest.private_key)) + platform_log: + enabled: true s3: environment: development region: ((region)) From e8b88089df287fde95c2231733cf66aa2172c97d Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 21 Apr 2026 13:00:57 -0400 Subject: [PATCH 113/165] only use platform objects --- opensearch-base.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 9e641b9..209ff65 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -269,10 +269,8 @@ instance_groups: - platform-shards-and-replica: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json - index-settings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-settings.json - index-mappings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-mappings.json - - component-index-mappings-base: /var/vcap/jobs/opensearch_templates/component-index-mappings-base.json - component-index-mappings-platform: /var/vcap/jobs/opensearch_templates/component-index-mappings-platform.json index_templates: - - index-template: /var/vcap/jobs/opensearch_templates/index-template.json - index-templat-platform: /var/vcap/jobs/opensearch_templates/index-template-platform.json release: opensearch - name: opensearch_templates From c3f466d653f8c362d31b086ba45d1fc169822e53 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 21 Apr 2026 14:07:24 -0400 Subject: [PATCH 114/165] removing old app componennts --- opensearch-base.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 209ff65..69c6421 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -276,18 +276,12 @@ instance_groups: - name: opensearch_templates properties: opensearch_config: - app_index_component_name: component-index-mappings-app-cf - app_index_settings: - index.mapping.total_fields.limit: 2000 - index.queries.cache.enabled: "false" - metric_index_settings: + platform_index_settings: index.mapping.total_fields.limit: 100 index.queries.cache.enabled: "false" - base_index_component_name: component-index-mappings-base index_mappings_component_name: index-mappings index_settings_component_name: index-settings platform_index_component_name: index-mappings-platform - metric_index_component_name: component-index-mappings-metrics platform_shards_and_replicas_component_name: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json release: opensearch - name: upload-dashboards-objects From 7e276ea728f4b3ddc402286cf30b83bae01ca06b Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 21 Apr 2026 16:03:35 -0400 Subject: [PATCH 115/165] removing old app componennts --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 69c6421..2d4bb61 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -281,7 +281,7 @@ instance_groups: index.queries.cache.enabled: "false" index_mappings_component_name: index-mappings index_settings_component_name: index-settings - platform_index_component_name: index-mappings-platform + platform_index_component_name: component-index-mappings-platform.json platform_shards_and_replicas_component_name: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json release: opensearch - name: upload-dashboards-objects From 0fbba90c148b2fd389bc6deeeca17f93c00a99e6 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Apr 2026 10:34:00 -0400 Subject: [PATCH 116/165] giving template opensearch --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 2d4bb61..65ef09d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -274,6 +274,7 @@ instance_groups: - index-templat-platform: /var/vcap/jobs/opensearch_templates/index-template-platform.json release: opensearch - name: opensearch_templates + consumes: *consumes-opensearch-manager properties: opensearch_config: platform_index_settings: From 9abc0bb8ec80a61d9ca820ce6fa6bf3b3270fb64 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Apr 2026 11:39:49 -0400 Subject: [PATCH 117/165] giving template opensearch --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 65ef09d..f90bd0a 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -271,7 +271,7 @@ instance_groups: - index-mappings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-mappings.json - component-index-mappings-platform: /var/vcap/jobs/opensearch_templates/component-index-mappings-platform.json index_templates: - - index-templat-platform: /var/vcap/jobs/opensearch_templates/index-template-platform.json + - index-template-platform: /var/vcap/jobs/opensearch_templates/index-template-platform.json release: opensearch - name: opensearch_templates consumes: *consumes-opensearch-manager From c1860ed62fcd882b1f7061897d20860af42bbd46 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Apr 2026 12:54:50 -0400 Subject: [PATCH 118/165] changing default --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index f90bd0a..194fc90 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -449,7 +449,7 @@ instance_groups: ssl: enabled: false port: 5605 - defaultAppId: dashboard/App-Overview + defaultAppId: dashboard/platform-overview-dashboard env: - NODE_ENV: production health: From 7fcde7c7d499adca9f57e70983af5900218d9e55 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Apr 2026 13:02:08 -0400 Subject: [PATCH 119/165] changing default --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 194fc90..99e7a84 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -282,7 +282,7 @@ instance_groups: index.queries.cache.enabled: "false" index_mappings_component_name: index-mappings index_settings_component_name: index-settings - platform_index_component_name: component-index-mappings-platform.json + platform_index_component_name: component-index-mappings-platform platform_shards_and_replicas_component_name: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json release: opensearch - name: upload-dashboards-objects From 25cf92e636994ffa7320aa8b381805b18b5d0d95 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Apr 2026 14:47:02 -0400 Subject: [PATCH 120/165] file name --- opensearch-base.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 99e7a84..3244df7 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -77,7 +77,7 @@ instance_groups: persistent_disk_type: logs_opensearch_os_old_data stemcell: default azs: [z1,z2] - vm_type: t3.large\ + vm_type: t3.large vm_extensions: - logs-opensearch-profile networks: @@ -283,7 +283,7 @@ instance_groups: index_mappings_component_name: index-mappings index_settings_component_name: index-settings platform_index_component_name: component-index-mappings-platform - platform_shards_and_replicas_component_name: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json + platform_shards_and_replicas_component_name: shards-and-replicas-platform release: opensearch - name: upload-dashboards-objects release: opensearch From 65b96cc8a4cac6780e486e287cfe009fa005aee1 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Apr 2026 15:01:59 -0400 Subject: [PATCH 121/165] file name --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 3244df7..0170ad4 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -283,7 +283,7 @@ instance_groups: index_mappings_component_name: index-mappings index_settings_component_name: index-settings platform_index_component_name: component-index-mappings-platform - platform_shards_and_replicas_component_name: shards-and-replicas-platform + platform_shards_and_replicas_component_name: platform-shards-and-replicas release: opensearch - name: upload-dashboards-objects release: opensearch From 729172cf5d564eb779d37ffde850ffce1d8799e6 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 22 Apr 2026 15:08:11 -0400 Subject: [PATCH 122/165] file name --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 0170ad4..566b354 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -283,7 +283,7 @@ instance_groups: index_mappings_component_name: index-mappings index_settings_component_name: index-settings platform_index_component_name: component-index-mappings-platform - platform_shards_and_replicas_component_name: platform-shards-and-replicas + platform_shards_and_replicas_component_name: platform-shards-and-replica release: opensearch - name: upload-dashboards-objects release: opensearch From f87aa7ac2e294d80a76e071438f47391a7dce741 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 23 Apr 2026 10:17:20 -0400 Subject: [PATCH 123/165] changing to use platform as default --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 566b354..a8224a8 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -302,6 +302,7 @@ instance_groups: - Falco client_id: opensearch_client_id dashboards_objects: + default_index: logs-platform* upload_patterns: - {type: index-pattern, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/index-pattern/logs-app*.json"} - {type: search, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/search/app-*.json"} From 7cdb0e098c7b3ee8b04027f687c49b4161dfad58 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 23 Apr 2026 10:37:42 -0400 Subject: [PATCH 124/165] changing to use platform as default --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index a8224a8..3ccbf8d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -302,7 +302,7 @@ instance_groups: - Falco client_id: opensearch_client_id dashboards_objects: - default_index: logs-platform* + default_index: logs-platform-* upload_patterns: - {type: index-pattern, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/index-pattern/logs-app*.json"} - {type: search, pattern: "/var/vcap/jobs/upload-dashboards-objects/dashboards-objects/search/app-*.json"} From fbd0495ba467a11d80283be971ba2e635ea356dd Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 23 Apr 2026 11:54:31 -0400 Subject: [PATCH 125/165] attempting s3 --- opensearch-base.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 3ccbf8d..dbebebc 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -366,17 +366,17 @@ instance_groups: filters: - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-platform-default.conf outputs: - # - plugin: s3 - # options: - # region: ((region)) - # bucket: ((bucket)) - # server_side_encryption: true - # prefix: "%{+yyyy/MM/dd/HH/mm}" - # encoding: "gzip" - # temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp - # # note that this is different than in the archivers - # # here, we're uploading the parsed message as json, not the raw log line - # codec: json + - plugin: s3 + options: + region: ((region)) + bucket: ((bucket)) + server_side_encryption: true + prefix: "%{+yyyy/MM/dd/HH/mm}" + encoding: "gzip" + temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp + # note that this is different than in the archivers + # here, we're uploading the parsed message as json, not the raw log line + codec: json - plugin: opensearch options: {} deployment_dictionary: From 0a24d28ce656c721c8e0b49e1432f8ca95d8d639 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 23 Apr 2026 13:03:01 -0400 Subject: [PATCH 126/165] adding back in base --- opensearch-base.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index dbebebc..3500f0b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -269,8 +269,10 @@ instance_groups: - platform-shards-and-replica: /var/vcap/jobs/upload_opensearch_config/index-templates/shards-and-replicas-platform.json - index-settings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-settings.json - index-mappings: /var/vcap/jobs/upload_opensearch_config/index-templates/index-mappings.json + - component-index-mappings-base: /var/vcap/jobs/opensearch_templates/component-index-mappings-base.json - component-index-mappings-platform: /var/vcap/jobs/opensearch_templates/component-index-mappings-platform.json index_templates: + - index-template: /var/vcap/jobs/opensearch_templates/index-template.json - index-template-platform: /var/vcap/jobs/opensearch_templates/index-template-platform.json release: opensearch - name: opensearch_templates @@ -398,6 +400,7 @@ instance_groups: release: opensearch vm_type: t3.medium vm_extensions: + - platform-syslog-lb - 20GB_ephemeral_disk persistent_disk_type: logs_opensearch_ingestor From a33c0e59847a0bf9fd5e77efc940adb0dced545c Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 23 Apr 2026 13:20:20 -0400 Subject: [PATCH 127/165] adding in base --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 3500f0b..21ea882 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -282,6 +282,7 @@ instance_groups: platform_index_settings: index.mapping.total_fields.limit: 100 index.queries.cache.enabled: "false" + base_index_component_name: component-index-mappings-base index_mappings_component_name: index-mappings index_settings_component_name: index-settings platform_index_component_name: component-index-mappings-platform From 94ba569decaa3d8ad5f6c77e70668c45b769be49 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 23 Apr 2026 15:52:47 -0400 Subject: [PATCH 128/165] adding in base --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 21ea882..dd84042 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -401,7 +401,7 @@ instance_groups: release: opensearch vm_type: t3.medium vm_extensions: - + - platform-opensearch-ingestor-profile - platform-syslog-lb - 20GB_ephemeral_disk persistent_disk_type: logs_opensearch_ingestor From bf7c8374b22e28c9228c547c2f41c3f50b1e28aa Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 27 Apr 2026 12:07:19 -0400 Subject: [PATCH 129/165] staging startup --- ci/pipeline.yml | 119 +++++++++++++++++++++++++++++-- opensearch-scaling-staging.yml | 125 +++++++++++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 6 deletions(-) create mode 100644 opensearch-scaling-staging.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 062c682..8714082 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -4,6 +4,8 @@ groups: jobs: - build-opensearch-test-release - deploy-opensearch-development + - upload-dashboards-objects-development + - smoke-tests-development jobs: - name: build-opensearch-test-release @@ -48,6 +50,7 @@ jobs: # channel: ((slack-channel-customer-success)) # username: ((slack-username)) # icon_url: ((slack-icon-url)) + - name: deploy-opensearch-development serial_groups: [bosh-development] plan: @@ -84,9 +87,7 @@ jobs: file: deploy-logs-platform-config/ci/terraform-secrets.yml - put: opensearch-development-deployment params: - dry_run: true manifest: opensearch-manifest/manifest.yml - # dry_run: true releases: - opensearch-test-release/*.tgz stemcells: @@ -96,9 +97,6 @@ jobs: - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml - - deploy-logs-platform-config/opsfiles/enable-ingestor-dns.yml - # - deploy-logs-platform-config/opsfiles/enable-proxy-auth.yml - # - deploy-logs-platform-config/opsfiles/enable-auth-proxy-route-dev.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml - deploy-logs-platform-config/opsfiles/cf-development.yml - deploy-logs-platform-config/opsfiles/opensearch-notification.yml @@ -112,6 +110,105 @@ jobs: # text: | # :x: FAILED to deploy logs-platform in development # <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + + - name: upload-dashboards-objects-development + serial_groups: [bosh-development] + plan: + - in_parallel: + - get: pipeline-tasks + - get: opensearch-test-release + - get: opensearch-stemcell-jammy + - get: deploy-logs-platform-config + - get: opensearch-development-deployment + trigger: true + - task: upload-dashboards-objects + file: pipeline-tasks/bosh-logs-errand.yml + params: + BOSH_ENVIRONMENT: ((bosh_development_environment)) + BOSH_CLIENT: ((bosh_client)) + BOSH_CLIENT_SECRET: ((bosh_development_client_secret)) + BOSH_DEPLOYMENT: logs-platform + BOSH_ERRAND: upload-dashboards-objects + BOSH_FLAGS: "--keep-alive" + BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) + + - name: smoke-tests-development + serial_groups: [bosh-development] + plan: + - in_parallel: + - get: tests-timer + - get: pipeline-tasks + - get: opensearch-test-release + - get: opensearch-stemcell-jammy + - get: deploy-logs-platform-config + - get: opensearch-development-deployment + trigger: true + - task: smoke-tests + file: pipeline-tasks/bosh-logs-errand.yml + params: + BOSH_ENVIRONMENT: ((bosh_development_environment)) + BOSH_CLIENT: ((bosh_client)) + BOSH_CLIENT_SECRET: ((bosh_development_client_secret)) + BOSH_DEPLOYMENT: logs-platform + BOSH_ERRAND: smoke_tests + BOSH_FLAGS: "--keep-alive" + BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) + + - name: deploy-opensearch-staging + serial_groups: [bosh-staging] + plan: + - in_parallel: + - get: pipeline-tasks + - get: deploy-logs-platform-config + passed: [smoke-tests-development] + trigger: true + - get: opensearch-test-release + passed: [smoke-tests-development] + trigger: true + - get: opensearch-stemcell-jammy + passed: [smoke-tests-development] + trigger: true + - get: general-task + - get: terraform-yaml + resource: terraform-yaml-staging + trigger: true + - task: opensearch-manifest + image: general-task + config: + platform: linux + inputs: + - name: deploy-logs-platform-config + run: + path: sh + args: + - -exc + - | + bosh int deploy-logs-platform-config/opensearch-base.yml \ + -o deploy-logs-platform-config/opensearch-scaling-staging.yml \ + > opensearch-manifest/manifest.yml + outputs: + - name: opensearch-manifest + - task: terraform-secrets + image: general-task + file: deploy-logs-platform-config/ci/terraform-secrets.yml + - put: opensearch-staging-deployment + params: + manifest: opensearch-manifest/manifest.yml + releases: + - opensearch-test-release/*.tgz + stemcells: + - opensearch-stemcell-jammy/*.tgz + ops_files: + - deploy-logs-platform-config/opsfiles/enable-node-tls.yml + - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml + - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml + - deploy-logs-platform-config/opsfiles/enable-syslog.yml + - deploy-logs-platform-config/opsfiles/cf-staging.yml + - deploy-logs-platform-config/opsfiles/opensearch-notification.yml + - deploy-logs-platform-config/opsfiles/add-keystore-credentials-errand.yml + vars_files: + - terraform-secrets/terraform.yml resources: - name: opensearch-test-release-git-repo @@ -149,16 +246,24 @@ resources: bucket: ((tf-state-bucket-development)) versioned_file: ((tf-state-file-development)) region_name: ((aws-region)) + + - name: tests-timer + type: time + source: + interval: 30m + - name: deploy-logs-platform-config type: git source: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/deploy-platform-opensearch.git branch: platform + - name: opensearch-stemcell-jammy source: name: bosh-aws-xen-hvm-ubuntu-jammy-go_agent type: bosh-io-stemcell + - name: pipeline-tasks type: git source: @@ -195,6 +300,7 @@ resource_types: repository: registry-image-resource aws_region: us-gov-west-1 tag: latest + # - name: slack-notification # type: registry-image # source: @@ -203,6 +309,7 @@ resource_types: # repository: slack-notification-resource # aws_region: us-gov-west-1 # tag: latest + - name: bosh-deployment type: registry-image source: @@ -226,4 +333,4 @@ resource_types: aws_secret_access_key: ((ecr_aws_secret)) repository: git-resource aws_region: us-gov-west-1 - tag: latest \ No newline at end of file + tag: latest diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml new file mode 100644 index 0000000..1d690a3 --- /dev/null +++ b/opensearch-scaling-staging.yml @@ -0,0 +1,125 @@ +- type: replace + path: /instance_groups/name=opensearch_manager?/vm_type? + value: t3.large + +- type: replace + path: /instance_groups/name=opensearch_manager?/instances? + value: 1 + + +- type: replace + path: /instance_groups/name=opensearch_data?/instances? + value: 2 + +- type: replace + path: /instance_groups/name=opensearch_data?/vm_type? + value: r6i.xlarge + +- type: replace + path: /instance_groups/name=opensearch_data?/update? + value: + max_in_flight: 1 + canaries: 1 + serial: true + + +- type: replace + path: /instance_groups/name=opensearch_old_data?/instances? + value: 4 + +- type: replace + path: /instance_groups/name=opensearch_old_data?/vm_type? + value: r6i.xlarge + +- type: replace + path: /instance_groups/name=opensearch_old_data?/update? + value: + max_in_flight: 1 + canaries: 1 + serial: true + + +- type: replace + path: /instance_groups/name=opensearch_dashboards?/instances? + value: 1 + +- type: replace + path: /instance_groups/name=opensearch_dashboards?/vm_type? + value: t3.xlarge + +- type: replace + path: /instance_groups/name=opensearch_dashboards/update? + value: + max_in_flight: 100% + canaries: 100% + serial: true + + +- type: replace + path: /instance_groups/name=ingestor?/instances? + value: 1 + +- type: replace + path: /instance_groups/name=ingestor?/vm_type? + value: m6i.large + + + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? + value: + index_pattern: logs-app-* + minimum: 1000 + run: false + time_field: '@timestamp' + time_interval: 5m + + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_document_size? + value: 3gb + + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_index_age? + value: 1d + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/shard_count? + value: 2 + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/metrics_shard_count? + value: 1 + + + +- type: replace + path: /instance_groups/name=maintenance?/instances? + value: 1 + +- type: replace + path: /instance_groups/name=maintenance?/vm_type? + value: t3.large + + + + +- type: replace + path: /addons? + value: + - name: bosh-dns-aliases + jobs: + - name: bosh-dns-aliases + release: bosh-dns-aliases + properties: + aliases: + - domain: nats.service.cf.internal + targets: + - deployment: cf-development + domain: bosh + instance_group: nats + network: default + query: '*' + + From 640b2ccc6eebbc6bf33df7f87b13e75e7c1e938f Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 27 Apr 2026 12:26:36 -0400 Subject: [PATCH 130/165] staging addition and some cleanup --- opensearch-base.yml | 16 ---------------- opensearch-scaling-development.yml | 6 +++--- opensearch-scaling-staging.yml | 12 +++--------- 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index dd84042..a33af36 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -237,13 +237,6 @@ instance_groups: smoke_tests: app_log: enabled: false - count_test: - index_pattern: logs-platform-* - minimum: 10 - long_time_interval: 10m - run: true - time_field: '@timestamp' - time_interval: 5m index: logs-platform opensearch: ssl: @@ -295,15 +288,6 @@ instance_groups: from: opensearch_manager ip_addresses: true properties: - cloudfoundry: - event_types: - - HttpStartStop - - LogMessage - - ContainerMetric - - Audit - - Metrics - - Falco - client_id: opensearch_client_id dashboards_objects: default_index: logs-platform-* upload_patterns: diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index b43da45..3082435 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -68,9 +68,9 @@ - type: replace path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? value: - index_pattern: logs-app-* - minimum: 1000 - run: false + index_pattern: logs-platform* + minimum: 100 + run: true time_field: '@timestamp' time_interval: 5m diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml index 1d690a3..d6c76b2 100644 --- a/opensearch-scaling-staging.yml +++ b/opensearch-scaling-staging.yml @@ -68,9 +68,9 @@ - type: replace path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? value: - index_pattern: logs-app-* + index_pattern: logs-platform-* minimum: 1000 - run: false + run: true time_field: '@timestamp' time_interval: 5m @@ -85,15 +85,9 @@ value: 1d - type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/shard_count? + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/platform_shard_count? value: 2 -- type: replace - path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/metrics_shard_count? - value: 1 - - - - type: replace path: /instance_groups/name=maintenance?/instances? value: 1 From ac021a69159ad44fc86bc016a65ddbbcb8077ec7 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Apr 2026 09:02:23 -0400 Subject: [PATCH 131/165] making all alias use the platform alias --- ci/pipeline.yml | 24 ++++++++++++++++++++++-- opensearch-base.yml | 2 +- opsfiles/cf-development.yml | 21 --------------------- opsfiles/cf-production.yml | 22 ---------------------- opsfiles/cf-staging.yml | 22 ---------------------- 5 files changed, 23 insertions(+), 68 deletions(-) delete mode 100644 opsfiles/cf-development.yml delete mode 100644 opsfiles/cf-production.yml delete mode 100644 opsfiles/cf-staging.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 8714082..c7751ea 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -6,6 +6,9 @@ groups: - deploy-opensearch-development - upload-dashboards-objects-development - smoke-tests-development + - name: staging + jobs: + - deploy-opensearch-staging jobs: - name: build-opensearch-test-release @@ -98,7 +101,6 @@ jobs: - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml - - deploy-logs-platform-config/opsfiles/cf-development.yml - deploy-logs-platform-config/opsfiles/opensearch-notification.yml - deploy-logs-platform-config/opsfiles/add-keystore-credentials-errand.yml vars_files: @@ -204,7 +206,6 @@ jobs: - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml - deploy-logs-platform-config/opsfiles/enable-syslog.yml - - deploy-logs-platform-config/opsfiles/cf-staging.yml - deploy-logs-platform-config/opsfiles/opensearch-notification.yml - deploy-logs-platform-config/opsfiles/add-keystore-credentials-errand.yml vars_files: @@ -247,6 +248,13 @@ resources: versioned_file: ((tf-state-file-development)) region_name: ((aws-region)) + - name: terraform-yaml-staging + type: s3-iam + source: + bucket: ((tf-state-bucket-staging)) + versioned_file: ((tf-state-file-staging)) + region_name: ((aws-region)) + - name: tests-timer type: time source: @@ -282,6 +290,16 @@ resources: client_secret: ((bosh-director-info.development.client_secret)) ca_cert: ((bosh-director-info.development.ca_cert)) deployment: logs-platform + + - name: opensearch-staging-deployment + type: bosh-deployment + source: &bosh-params-staging + target: ((bosh-director-info.staging.environment)) + client: ((bosh-director-info.staging.client)) + client_secret: ((bosh-director-info.staging.client_secret)) + ca_cert: ((bosh-director-info.staging.ca_cert)) + deployment: logs-platform + - name: general-task type: registry-image source: @@ -318,6 +336,7 @@ resource_types: repository: bosh-deployment-resource aws_region: us-gov-west-1 tag: latest + - name: s3-iam type: registry-image source: @@ -326,6 +345,7 @@ resource_types: repository: s3-resource aws_region: us-gov-west-1 tag: latest + - name: git type: registry-image source: diff --git a/opensearch-base.yml b/opensearch-base.yml index a33af36..1c8da78 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -344,7 +344,7 @@ instance_groups: opensearch: data_hosts: - localhost - index: ((alias)) + index: ((platform_alias)) index_type: '%{@type}' ssl: ca: ((opensearch_node.ca)) diff --git a/opsfiles/cf-development.yml b/opsfiles/cf-development.yml deleted file mode 100644 index 9a9b6e1..0000000 --- a/opsfiles/cf-development.yml +++ /dev/null @@ -1,21 +0,0 @@ -# replace for client_password -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/client_password? - value: ((/bosh/cf-development/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/system_domain? - value: ((cf-api-development)) - -- type: replace - path: /instance_groups/name=opensearch_data/jobs/name=opensearch/properties?/opensearch?/cf?/client_password? - value: ((/bosh/cf-development/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=opensearch_data/jobs/name=opensearch/properties?/opensearch?/cf?/domain? - value: ((cf-api-development)) - -- type: replace - path: /instance_groups/name=opensearch_old_data/jobs/name=opensearch/properties?/opensearch?/cf?/client_password? - value: ((/bosh/cf-development/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=opensearch_old_data/jobs/name=opensearch/properties?/opensearch?/cf?/domain? - value: ((cf-api-development)) diff --git a/opsfiles/cf-production.yml b/opsfiles/cf-production.yml deleted file mode 100644 index 17cce14..0000000 --- a/opsfiles/cf-production.yml +++ /dev/null @@ -1,22 +0,0 @@ -# replace for client_password - -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/client_password? - value: ((/bosh/cf-production/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/system_domain? - value: ((cf-api-production)) - -- type: replace - path: /instance_groups/name=opensearch_data/jobs/name=opensearch/properties?/opensearch?/cf?/client_password? - value: ((/bosh/cf-production/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=opensearch_data/jobs/name=opensearch/properties?/opensearch?/cf?/domain? - value: ((cf-api-production)) - -- type: replace - path: /instance_groups/name=opensearch_old_data/jobs/name=opensearch/properties?/opensearch?/cf?/client_password? - value: ((/bosh/cf-production/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=opensearch_old_data/jobs/name=opensearch/properties?/opensearch?/cf?/domain? - value: ((cf-api-production)) diff --git a/opsfiles/cf-staging.yml b/opsfiles/cf-staging.yml deleted file mode 100644 index 0e6d25d..0000000 --- a/opsfiles/cf-staging.yml +++ /dev/null @@ -1,22 +0,0 @@ -# replace for client_password - -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/client_password? - value: ((/bosh/cf-staging/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=maintenance/jobs/name=upload-dashboards-objects/properties?/cloudfoundry?/system_domain? - value: ((cf-api-staging)) - -- type: replace - path: /instance_groups/name=opensearch_data/jobs/name=opensearch/properties?/opensearch?/cf?/client_password? - value: ((/bosh/cf-staging/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=opensearch_data/jobs/name=opensearch/properties?/opensearch?/cf?/domain? - value: ((cf-api-staging)) - -- type: replace - path: /instance_groups/name=opensearch_old_data/jobs/name=opensearch/properties?/opensearch?/cf?/client_password? - value: ((/bosh/cf-staging/opensearch_client_secret)) -- type: replace - path: /instance_groups/name=opensearch_old_data/jobs/name=opensearch/properties?/opensearch?/cf?/domain? - value: ((cf-api-staging)) From 4d635d300dddfce939a896f2648c11cf13299edf Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Apr 2026 09:31:14 -0400 Subject: [PATCH 132/165] setting to 0 at first --- opensearch-scaling-staging.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml index d6c76b2..2791ef9 100644 --- a/opensearch-scaling-staging.yml +++ b/opensearch-scaling-staging.yml @@ -9,7 +9,8 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 0 + #2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? @@ -25,7 +26,8 @@ - type: replace path: /instance_groups/name=opensearch_old_data?/instances? - value: 4 + value: 0 + # 4 - type: replace path: /instance_groups/name=opensearch_old_data?/vm_type? From ce4e603becc98615a787641fe2997bbdb31eb0d9 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Apr 2026 10:53:28 -0400 Subject: [PATCH 133/165] Dry run --- ci/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index c7751ea..0e92ebf 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -195,6 +195,7 @@ jobs: file: deploy-logs-platform-config/ci/terraform-secrets.yml - put: opensearch-staging-deployment params: + dry_run: true manifest: opensearch-manifest/manifest.yml releases: - opensearch-test-release/*.tgz From 522ce22daa7bb99d814e3c5bb8424e5ac7a90cd1 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Apr 2026 12:07:37 -0400 Subject: [PATCH 134/165] Dry run --- ci/pipeline.yml | 1 - opensearch-scaling-staging.yml | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 0e92ebf..c7751ea 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -195,7 +195,6 @@ jobs: file: deploy-logs-platform-config/ci/terraform-secrets.yml - put: opensearch-staging-deployment params: - dry_run: true manifest: opensearch-manifest/manifest.yml releases: - opensearch-test-release/*.tgz diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml index 2791ef9..d6c76b2 100644 --- a/opensearch-scaling-staging.yml +++ b/opensearch-scaling-staging.yml @@ -9,8 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 - #2 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? @@ -26,8 +25,7 @@ - type: replace path: /instance_groups/name=opensearch_old_data?/instances? - value: 0 - # 4 + value: 4 - type: replace path: /instance_groups/name=opensearch_old_data?/vm_type? From a8af4cfeaf80c6c1f338fb916d751104082ad322 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Apr 2026 12:41:13 -0400 Subject: [PATCH 135/165] removing old things from logs and adding in staging jobs --- ci/e2e-tests.yml | 12 - ci/pipeline.yml | 44 ++ ci/provision-test-user-cf-access.sh | 133 ------ ci/update-test-user-passwords.sh | 45 -- ci/upload-audit-events-s3.yml | 19 - ci/upload_audit_events_s3.py | 177 -------- e2e/__init__.py | 39 -- e2e/conftest.py | 60 --- e2e/notifications.py | 349 --------------- e2e/test_alerting_access.py | 418 ------------------ e2e/test_smoke_tests_login.py | 16 - e2e/user.py | 9 - e2e/utils.py | 179 -------- opsfiles/enable-auth-proxy-route-dev.yml | 11 - .../enable-auth-proxy-route-production.yml | 16 - opsfiles/enable-auth-proxy-route-staging.yml | 16 - opsfiles/enable-auth-proxy-route-test.yml | 16 - opsfiles/enable-proxy-auth.yml | 119 ----- scripts/download-e2e-ci-results.sh | 24 - scripts/e2e-local.sh | 11 - scripts/e2e.sh | 7 - varsfiles/logsearch_ingestor | 83 ++++ varsfiles/opensearach_ingestor | 93 ++++ 23 files changed, 220 insertions(+), 1676 deletions(-) delete mode 100644 ci/e2e-tests.yml delete mode 100755 ci/provision-test-user-cf-access.sh delete mode 100755 ci/update-test-user-passwords.sh delete mode 100644 ci/upload-audit-events-s3.yml delete mode 100755 ci/upload_audit_events_s3.py delete mode 100644 e2e/__init__.py delete mode 100644 e2e/conftest.py delete mode 100644 e2e/notifications.py delete mode 100644 e2e/test_alerting_access.py delete mode 100644 e2e/test_smoke_tests_login.py delete mode 100644 e2e/user.py delete mode 100644 e2e/utils.py delete mode 100644 opsfiles/enable-auth-proxy-route-dev.yml delete mode 100644 opsfiles/enable-auth-proxy-route-production.yml delete mode 100644 opsfiles/enable-auth-proxy-route-staging.yml delete mode 100644 opsfiles/enable-auth-proxy-route-test.yml delete mode 100644 opsfiles/enable-proxy-auth.yml delete mode 100755 scripts/download-e2e-ci-results.sh delete mode 100755 scripts/e2e-local.sh delete mode 100755 scripts/e2e.sh create mode 100644 varsfiles/logsearch_ingestor create mode 100644 varsfiles/opensearach_ingestor diff --git a/ci/e2e-tests.yml b/ci/e2e-tests.yml deleted file mode 100644 index 759f99f..0000000 --- a/ci/e2e-tests.yml +++ /dev/null @@ -1,12 +0,0 @@ -platform: linux - -inputs: -- name: deploy-logs-platform-config - -run: - path: sh - args: - - -exc - - | - cd deploy-logs-platform-config - ./scripts/e2e.sh diff --git a/ci/pipeline.yml b/ci/pipeline.yml index c7751ea..2181d97 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -211,6 +211,50 @@ jobs: vars_files: - terraform-secrets/terraform.yml + - name: upload-dashboards-objects-staging + serial_groups: [bosh-staging] + plan: + - in_parallel: + - get: pipeline-tasks + - get: opensearch-test-release + - get: opensearch-stemcell-jammy + - get: deploy-logs-platform-config + - get: opensearch-staging-deployment + trigger: true + - task: upload-dashboards-objects + file: pipeline-tasks/bosh-logs-errand.yml + params: + BOSH_ENVIRONMENT: ((bosh_staging_environment)) + BOSH_CLIENT: ((bosh_client)) + BOSH_CLIENT_SECRET: ((bosh_staging_client_secret)) + BOSH_DEPLOYMENT: logs-platform + BOSH_ERRAND: upload-dashboards-objects + BOSH_FLAGS: "--keep-alive" + BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) + + - name: smoke-tests-staging + serial_groups: [bosh-staging] + plan: + - in_parallel: + - get: tests-timer + - get: pipeline-tasks + - get: opensearch-test-release + - get: opensearch-stemcell-jammy + - get: deploy-logs-platform-config + - get: opensearch-staging-deployment + trigger: true + - task: smoke-tests + file: pipeline-tasks/bosh-logs-errand.yml + params: + BOSH_ENVIRONMENT: ((bosh_staging_environment)) + BOSH_CLIENT: ((bosh_client)) + BOSH_CLIENT_SECRET: ((bosh_staging_client_secret)) + BOSH_DEPLOYMENT: logs-platform + BOSH_ERRAND: smoke_tests + BOSH_FLAGS: "--keep-alive" + BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) + + resources: - name: opensearch-test-release-git-repo type: git diff --git a/ci/provision-test-user-cf-access.sh b/ci/provision-test-user-cf-access.sh deleted file mode 100755 index 3f3599c..0000000 --- a/ci/provision-test-user-cf-access.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if ! cf target > /dev/null; then - cf api "${CF_API_URL}" - cf auth -fi - -required_env_vars=( - TEST_USER_1_USERNAME - TEST_USER_2_USERNAME - TEST_USER_3_USERNAME - TEST_USER_4_USERNAME - CF_ORG_1_NAME - CF_ORG_2_NAME - CF_ORG_3_NAME -) -for var in "${required_env_vars[@]}"; do - if [ -z "${!var+x}" ]; then - echo "$var is a required environment variable" - exit 1 - fi -done - -function escape_username() { - USERNAME_ESCAPED=$(echo "$1" | jq -Rr @uri) - echo "$USERNAME_ESCAPED" -} - -function get_user_guid() { - USERNAME_ESCAPED=$(escape_username "$1") - USER_GUID=$(cf curl "/v3/users?partial_usernames=$USERNAME_ESCAPED" | jq -er '.resources[0].guid') - echo "$USER_GUID" -} - -function set_org_user() { - USER_GUID=$(get_user_guid "$1") - ORG_GUID=$(cf org "$2" --guid) - TMP_FILE=$(mktemp) - - COUNT_ORG_USER_ROLES=$(cf curl "/v3/roles?organization_guids=$ORG_GUID&user_guids=$USER_GUID&types=organization_user" | jq -r '.pagination.total_results') - if [[ $COUNT_ORG_USER_ROLES -gt 0 ]]; then - echo "user already has organization_user role in $2, continuing" - return - fi - - cat > "${TMP_FILE}" << EOF -{ - "type": "organization_user", - "relationships": { - "user": { - "data": { - "guid": "$USER_GUID" - } - }, - "organization": { - "data": { - "guid": "$ORG_GUID" - } - } - } -} -EOF - - cf curl "/v3/roles" \ - -X POST \ - -d "@$TMP_FILE" \ - --fail > /dev/null - - rm "$TMP_FILE" -} - -function delete_sandbox_org_roles() { - SANDBOX_ORG="sandbox-$(echo "$1" | cut -d '@' -f2 | cut -d '.' -f1)" - if ! cf org "$SANDBOX_ORG" > /dev/null; then - return - fi - - SANDBOX_SPACE=$(echo "$1" | cut -d '@' -f1) - if ! cf space "$SANDBOX_SPACE" > /dev/null; then - return - fi - - SANDBOX_ORG_GUID=$(cf org "$SANDBOX_ORG" --guid) - USER_GUID=$(get_user_guid "$1") - - for space_role in SpaceManager SpaceDeveloper SpaceAuditor; do - cf unset-space-role "$1" "$SANDBOX_ORG" "$SANDBOX_SPACE" "$space_role" - done - - for role_guid in $(cf curl "/v3/roles?organization_guids=$SANDBOX_ORG_GUID&user_guids=$USER_GUID" | jq -r '.resources[].guid'); do - cf curl "/v3/roles/$role_guid" -X DELETE - done -} - -# Expected results: -# - User 1 is in org 1 and org 3 -# - User 2 is in org 2. User 2 shares no orgs with User 1. -# - User 3 is in org 1. User 3 shares 1 org with User 1. -# - User 4 is in org 1 and org 3. User 4 shares all orgs with User 1. - -cf create-org "$CF_ORG_1_NAME" -cf create-org "$CF_ORG_2_NAME" -cf create-org "$CF_ORG_3_NAME" - -# Delete sandbox org roles for test users so we can be sure that -# their only orgs are the ones we have provisioned -delete_sandbox_org_roles "$TEST_USER_1_USERNAME" -delete_sandbox_org_roles "$TEST_USER_2_USERNAME" -delete_sandbox_org_roles "$TEST_USER_3_USERNAME" -delete_sandbox_org_roles "$TEST_USER_4_USERNAME" - -# User 1 is an org manager in org 1 and org 3 -cf set-org-role "$TEST_USER_1_USERNAME" "$CF_ORG_1_NAME" OrgManager -set_org_user "$TEST_USER_1_USERNAME" "$CF_ORG_1_NAME" -cf set-org-role "$TEST_USER_1_USERNAME" "$CF_ORG_3_NAME" OrgManager -set_org_user "$TEST_USER_1_USERNAME" "$CF_ORG_3_NAME" - -# User 2 is an org manager in org 2 -cf set-org-role "$TEST_USER_2_USERNAME" "$CF_ORG_2_NAME" OrgManager -set_org_user "$TEST_USER_2_USERNAME" "$CF_ORG_2_NAME" - -# User 3 is an org manager in org 1 -cf set-org-role "$TEST_USER_3_USERNAME" "$CF_ORG_1_NAME" OrgManager -set_org_user "$TEST_USER_3_USERNAME" "$CF_ORG_1_NAME" - -# User 4 is an org manager in org 1 and org 3 -cf set-org-role "$TEST_USER_4_USERNAME" "$CF_ORG_1_NAME" OrgManager -set_org_user "$TEST_USER_4_USERNAME" "$CF_ORG_1_NAME" -cf set-org-role "$TEST_USER_4_USERNAME" "$CF_ORG_3_NAME" OrgManager -set_org_user "$TEST_USER_4_USERNAME" "$CF_ORG_3_NAME" - diff --git a/ci/update-test-user-passwords.sh b/ci/update-test-user-passwords.sh deleted file mode 100755 index ca48af5..0000000 --- a/ci/update-test-user-passwords.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Login to UAA -uaac target "$UAA_API_URL" -uaac token client get "$UAA_CLIENT_ID" -s "$UAA_CLIENT_SECRET" - -TEST_USER_CREDENTIAL_NAMES=$(echo "$TEST_USERS_CREDENTIAL_USERNAME_MAP" | jq '. | keys | join(" ")') - -for credential_name in $TEST_USER_CREDENTIAL_NAMES; do - credential_name=$(echo "$credential_name" | tr -d '"') - - printf "updating password credential for %s\n\n" "$credential_name" - - # Generate a new password for the credential - PASSWORD_CREDENTIAL="/concourse/main/deploy-logs-platform/$credential_name" - if ! credhub get -n "$PASSWORD_CREDENTIAL" > /dev/null; then - credhub generate -n "$PASSWORD_CREDENTIAL" --type password - else - credhub regenerate -n "$PASSWORD_CREDENTIAL" - fi - - # Get the UAA username for the corresponding Credhub credential - USERNAME=$(echo "$TEST_USERS_CREDENTIAL_USERNAME_MAP" | jq -r --arg credential_name "$credential_name" '.[$credential_name]') - - # Get the new password from Credhub - PASSWORD=$(credhub get -n "$PASSWORD_CREDENTIAL" --output-json | jq -r '.value') - - if ! uaac user get "$USERNAME" > /dev/null; then - printf "Creating UAA user %s\n\n" "$USERNAME" - uaac user add "$USERNAME" --password "$PASSWORD" --origin cloud.gov --emails "$USERNAME" - else - printf "updating UAA password for %s\n\n" "$USERNAME" - - # Update the user password in UAA with the new value from Credhub - uaac password set "$USERNAME" --password "$PASSWORD" - fi - - # Activate the user, just to be safe - uaac user activate "$USERNAME" -done - - - diff --git a/ci/upload-audit-events-s3.yml b/ci/upload-audit-events-s3.yml deleted file mode 100644 index 368da5c..0000000 --- a/ci/upload-audit-events-s3.yml +++ /dev/null @@ -1,19 +0,0 @@ -platform: linux - -inputs: -- name: deploy-logs-platform-config - -run: - path: sh - args: - - -c - - | - cd deploy-logs-platform-config/ - python -m pip install -r requirements-test.txt - python ci/upload_audit_events_s3.py -params: - AWS_DEFAULT_REGION: - AWS_ACCESS_KEY_ID: - AWS_SECRET_ACCESS_KEY: - BUCKET: - DEFAULT_TIME: diff --git a/ci/upload_audit_events_s3.py b/ci/upload_audit_events_s3.py deleted file mode 100755 index 071d817..0000000 --- a/ci/upload_audit_events_s3.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python - -from urllib.parse import urljoin -from botocore.exceptions import ClientError - -import json -import boto3 -import requests -import os -import functools -from datetime import datetime, timedelta, timezone - - -s3_client = boto3.client("s3") -timestamp_key = "timestamp" - - -class AuditEventsS3Uploader: - def __init__(self): - self.CF_API_URL = os.environ.get("CF_API_URL") - self.UAA_API_URL = os.environ.get("UAA_API_URL") - self.UAA_CLIENT_ID = os.environ.get("UAA_CLIENT_ID") - self.UAA_CLIENT_SECRET = os.environ.get("UAA_CLIENT_SECRET") - self.bucket_name = "{}".format(os.environ["BUCKET"]) - self.default_time = os.environ.get("DEFAULT_TIME") - self.token = self.get_client_credentials_token() - - def get_client_credentials_token(self): - with requests.Session() as s: - response = s.post( - urljoin(self.UAA_API_URL, "oauth/token"), - data={ - "grant_type": "client_credentials", - "client_id": self.UAA_CLIENT_ID, - "client_secret": self.UAA_CLIENT_SECRET, - "response_type": "token", - }, - auth=requests.auth.HTTPBasicAuth( - self.UAA_CLIENT_ID, self.UAA_CLIENT_SECRET - ), - timeout=30, - ) - response.raise_for_status() - return response.json()["access_token"] - - @functools.cache - def get_cf_entity_name(self, entity_path, entity_guid): - """ - Retrieves the name of a CF entity from a GUID. - """ - if not entity_guid: - return - - with requests.Session() as s: - s.headers["Authorization"] = f"Bearer {self.token}" - url = urljoin(self.CF_API_URL, f"v3/{entity_path}/{entity_guid}") - response = s.get(url) - - if response.status_code == 404: - return - - data = response.json() - return data["name"] - - def get_audit_logs(self, start, end): - audit_logs = [] - - with requests.Session() as s: - s.headers["Authorization"] = f"Bearer {self.token}" - params = { - "created_ats[gt]": str(start), - "created_ats[lt]": str(end), - "order_by": "created_at", - } - url = urljoin(self.CF_API_URL, "/v3/audit_events") - - first_response = s.get(url, params=params) - data = first_response.json() - audit_logs.extend(data["resources"]) - - while data["pagination"]["next"] is not None: - data = s.get(data["pagination"]["next"]["href"]).json() - audit_logs.extend(data["resources"]) - - return audit_logs - - def transform_audit_event(self, audit_event): - transformed_event = audit_event - - # remove "links" property from event - transformed_event.pop("links") - - if organization := audit_event.get("organization"): - if organization_name := self.get_cf_entity_name( - "organizations", - organization.get("guid", None), - ): - transformed_event["organization_name"] = organization_name - - if space := audit_event.get("space"): - if space_name := self.get_cf_entity_name("spaces", space.get("guid", None)): - transformed_event["space_name"] = space_name - - return transformed_event - - # Upload a batch of audit events to S3 as a single object - def put_audit_events_to_s3(self, object_name, audit_events): - body = "\n".join( - [ - json.dumps(self.transform_audit_event(audit_event)) - for audit_event in audit_events - ] - ) - s3_client.put_object( - Bucket=self.bucket_name, - Key=object_name, - Body=body, - ContentType="text/plain", - ServerSideEncryption='AES256' - ) - - def update_latest_stamp_in_s3(self, latest_timestamp): - data = latest_timestamp - s3_client.put_object( - Bucket=self.bucket_name, - Key=timestamp_key, - Body=data, - ServerSideEncryption='AES256' - ) - - def get_start_end_time(self, now): - time_ago = now - timedelta(minutes=int(self.default_time)) - start_time = time_ago.strftime("%Y-%m-%dT%H:%M:%SZ") - end_time = now.strftime("%Y-%m-%dT%H:%M:%SZ") - - try: - current_stamp_response = s3_client.get_object( - Bucket=self.bucket_name, Key=timestamp_key - ) - start_time = current_stamp_response["Body"].read().strip().decode("utf-8") - - except ClientError as e: - # There is no timestamp key yet - if e.response["Error"]["Code"] == "NoSuchKey": - print(f"No existing start timestamp, starting from {start_time}") - else: - raise e - return (start_time, end_time) - - def upload_audit_events_to_s3(self): - now = datetime.now(timezone.utc) - (start_time, end_time) = self.get_start_end_time(now) - - audit_logs = self.get_audit_logs(start_time, end_time) - if len(audit_logs) > 0: - timestamp = audit_logs[-1]["created_at"] - object_name = f"{now.year}/{now.month:02d}/{now.day:02d}/{now.hour:02d}/{now.minute:02d}/{now.second:02d}" - try: - self.put_audit_events_to_s3(object_name, audit_logs) - print(f"success for start time {start_time} and end time {end_time}") - except Exception as e: - print( - f"Error upload file to S3 for time starting {start_time} and end time {end_time}" - ) - raise e - self.update_latest_stamp_in_s3(timestamp) - else: - self.update_latest_stamp_in_s3(end_time) - - -def main(): - audit_events_s3_uploader = AuditEventsS3Uploader() - audit_events_s3_uploader.upload_audit_events_to_s3() - - -if __name__ == "__main__": - main() diff --git a/e2e/__init__.py b/e2e/__init__.py deleted file mode 100644 index 274488f..0000000 --- a/e2e/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import sys - -required_env_vars = [ - "AUTH_PROXY_URL", - "UAA_BASE_URL", - "CF_ORG_1_NAME", - "CF_ORG_2_NAME", - "CF_ORG_3_NAME", - "TEST_USER_1_USERNAME", - "TEST_USER_1_PASSWORD", - "TEST_USER_1_TOTP_SEED", - "TEST_USER_2_USERNAME", - "TEST_USER_2_PASSWORD", - "TEST_USER_2_TOTP_SEED", - "TEST_USER_3_USERNAME", - "TEST_USER_3_PASSWORD", - "TEST_USER_3_TOTP_SEED", - "TEST_USER_4_USERNAME", - "TEST_USER_4_PASSWORD", - "TEST_USER_4_TOTP_SEED", - "SMTP_SENDER_HOST", - "SMTP_SENDER_PORT", - "SMTP_SENDER_FROM", -] - -for env_var in required_env_vars: - if os.environ[env_var] is None: - print(f"{env_var} is a required environment variable, exiting") - sys.exit(1) - -AUTH_PROXY_URL = os.environ["AUTH_PROXY_URL"] -UAA_BASE_URL = os.environ["AUTH_PROXY_URL"] -CF_ORG_1_NAME = os.environ["CF_ORG_1_NAME"] -CF_ORG_2_NAME = os.environ["CF_ORG_2_NAME"] -CF_ORG_3_NAME = os.environ["CF_ORG_3_NAME"] -SMTP_SENDER_HOST = os.environ["SMTP_SENDER_HOST"] -SMTP_SENDER_PORT = os.environ["SMTP_SENDER_PORT"] -SMTP_SENDER_FROM = os.environ["SMTP_SENDER_FROM"] diff --git a/e2e/conftest.py b/e2e/conftest.py deleted file mode 100644 index 0343ac0..0000000 --- a/e2e/conftest.py +++ /dev/null @@ -1,60 +0,0 @@ -from os import getenv -from playwright.sync_api import expect - -import pytest - -from .user import User - -expect.set_options(timeout=60_000) - - -@pytest.fixture -def user_1(): - user_1_username = getenv(f"TEST_USER_1_USERNAME") - user_1_password = getenv(f"TEST_USER_1_PASSWORD") - user_1_totp_seed = getenv(f"TEST_USER_1_TOTP_SEED") - user_1 = User( - user_1_username, - user_1_password, - user_1_totp_seed, - ) - return user_1 - - -@pytest.fixture -def user_2(): - user_2_username = getenv(f"TEST_USER_2_USERNAME") - user_2_password = getenv(f"TEST_USER_2_PASSWORD") - user_2_totp_seed = getenv(f"TEST_USER_2_TOTP_SEED") - user_2 = User( - user_2_username, - user_2_password, - user_2_totp_seed, - ) - return user_2 - - -@pytest.fixture -def user_3(): - user_3_username = getenv(f"TEST_USER_3_USERNAME") - user_3_password = getenv(f"TEST_USER_3_PASSWORD") - user_3_totp_seed = getenv(f"TEST_USER_3_TOTP_SEED") - user_3 = User( - user_3_username, - user_3_password, - user_3_totp_seed, - ) - return user_3 - - -@pytest.fixture -def user_4(): - user_4_username = getenv(f"TEST_USER_4_USERNAME") - user_4_password = getenv(f"TEST_USER_4_PASSWORD") - user_4_totp_seed = getenv(f"TEST_USER_4_TOTP_SEED") - user_4 = User( - user_4_username, - user_4_password, - user_4_totp_seed, - ) - return user_4 diff --git a/e2e/notifications.py b/e2e/notifications.py deleted file mode 100644 index 61b4845..0000000 --- a/e2e/notifications.py +++ /dev/null @@ -1,349 +0,0 @@ -from playwright.sync_api import expect -import re - -from .utils import ( - wait_for_header, - click_delete_button, - fill_delete_confirm_placeholder, - click_delete_button, - select_table_item_checkbox, - open_actions_menu, - wait_for_loading_finished, - update_rows_per_table, - click_save_button, - dismiss_toast_notification_button, -) - -from . import SMTP_SENDER_HOST, SMTP_SENDER_PORT, SMTP_SENDER_FROM - - -def wait_for_channels_header(page): - wait_for_header(page, re.compile(r"^Channels\s\([0-9]+\)$")) - - -def fill_email_recipient_group_details(page, user, email_recipient_group_name): - group_name_input = page.get_by_placeholder("Enter recipient group name") - group_name_input.wait_for() - group_name_input.fill(email_recipient_group_name) - - email_address_input = ( - page.locator("div") - .filter(has_text=re.compile(r"^Email addresses$")) - .get_by_role("textbox") - ) - email_address_input.fill(user.username) - page.keyboard.press("Enter") - - -def create_email_recipient_group(page, user, email_recipient_group_name): - create_group_button = ( - page.locator("div") - .filter(has_text=re.compile(r"^Create recipient group$")) - .get_by_role("link") - ) - create_group_button.wait_for() - create_group_button.click() - - fill_email_recipient_group_details(page, user, email_recipient_group_name) - - create_group_button = page.get_by_role("button", name="Create") - create_group_button.wait_for() - create_group_button.click() - - wait_for_header(page, "Email recipient groups") - - update_rows_per_table(page) - - wait_for_loading_finished(page) - - expect(page.get_by_text(email_recipient_group_name, exact=True)).to_be_visible() - - -def fill_email_smtp_sender_details(page, email_sender_name): - sender_name_input = page.get_by_placeholder("Enter sender name") - sender_name_input.wait_for() - sender_name_input.fill(email_sender_name) - - sender_email_input = page.get_by_label("Email address") - sender_email_input.wait_for() - sender_email_input.fill(SMTP_SENDER_FROM) - - sender_host_input = page.get_by_label("Host") - sender_host_input.wait_for() - sender_host_input.fill(SMTP_SENDER_HOST) - - sender_host_input = page.get_by_label("Port") - sender_host_input.wait_for() - sender_host_input.fill(SMTP_SENDER_PORT) - - -def create_email_smtp_sender(page, email_sender_name): - create_group_button = ( - page.locator("div") - .filter(has_text=re.compile(r"^Create SMTP sender$")) - .get_by_role("link") - ) - create_group_button.wait_for() - create_group_button.click() - - fill_email_smtp_sender_details(page, email_sender_name) - - create_group_button = page.get_by_role("button", name="Create") - create_group_button.wait_for() - create_group_button.click() - - wait_for_header(page, "Email senders") - - update_rows_per_table(page) - - wait_for_loading_finished(page) - - expect(page.get_by_text(email_sender_name, exact=True)).to_be_visible() - - -def create_notifications_channel( - page, email_recipient_group_name, email_sender_name, channel_name -): - create_channel_button = ( - page.locator("div") - .filter(has_text=re.compile(r"^Create channel$")) - .get_by_role("link") - ) - create_channel_button.wait_for() - create_channel_button.click() - - channel_name_input = page.get_by_label("Name") - channel_name_input.wait_for() - channel_name_input.fill(channel_name) - - # this is probably brittle because it assumes that Slack is the - # default option for a channel, which could change. but the markup - # here is not accessible and easy to select otherwise - channel_type_button = page.get_by_role("button", name="Slack") - channel_type_button.wait_for() - channel_type_button.click() - - email_option = page.get_by_role("option", name="Email") - email_option.wait_for() - email_option.click() - - cloud_smtp_sender = page.get_by_role("textbox", name="SMTP sender") - cloud_smtp_sender.wait_for() - cloud_smtp_sender.fill(email_sender_name) - page.keyboard.press("Enter") - - enter_recipient_group_div = page.locator("div").filter( - has_text=re.compile(r"^Email address, recipient group name$") - ) - enter_recipients_placeholder = enter_recipient_group_div.first - enter_recipients_placeholder.wait_for() - enter_recipients_placeholder.click() - - recipient_group_input = page.get_by_label("Default recipients") - recipient_group_input.wait_for() - recipient_group_input.fill(email_recipient_group_name) - page.keyboard.press("Enter") - - create_channel_button = page.get_by_role("button", name="Create", exact=True) - create_channel_button.wait_for() - create_channel_button.click() - - wait_for_channels_header(page) - - expect(page.get_by_role("link", name=channel_name, exact=True)).to_be_visible() - - -def create_alert_monitor(page, monitor_name, trigger_name, action_name, channel_name): - create_monitor_button = page.get_by_role("link", name="Create monitor").first - create_monitor_button.wait_for() - create_monitor_button.click() - - monitor_name_input = page.locator('input[name="name"]') - monitor_name_input.wait_for() - monitor_name_input.fill(monitor_name) - - index_input = page.locator("#index") - index_input.wait_for() - index_input.fill("logs-app-*") - page.keyboard.press("Enter") - - wait_for_loading_finished(page) - - time_field_input = page.locator("#timeField") - time_field_input.wait_for() - time_field_input.fill("@timestamp") - page.keyboard.press("Enter") - - wait_for_loading_finished(page) - - add_trigger_button = page.get_by_role("button", name="Add trigger", exact=True) - add_trigger_button.wait_for() - add_trigger_button.click() - - wait_for_loading_finished(page) - - trigger_name_input = page.locator('input[name="triggerDefinitions[0].name"]') - trigger_name_input.wait_for() - trigger_name_input.fill(trigger_name) - - query_time_interval = page.locator('input[name="bucketValue"]') - query_time_interval.wait_for() - query_time_interval.fill("15") - - wait_for_loading_finished(page) - - query_time_interval_unit_select = page.locator("#bucketUnitOfTime") - query_time_interval_unit_select.wait_for() - query_time_interval_unit_select.select_option(label="minute(s)") - - wait_for_loading_finished(page) - - trigger_threshold_input = page.locator( - 'input[name="triggerDefinitions[0].thresholdValue"]' - ) - trigger_threshold_input.wait_for() - # set threshold to 1 billion records for alert to trigger so that e2e tests for - # access do not actually trigger alerts - trigger_threshold_input.fill("1000000000") - - wait_for_loading_finished(page) - - action_name_input = page.get_by_placeholder("Enter action name") - action_name_input.wait_for() - action_name_input.fill(action_name) - - select_channel_placeholder = ( - page.locator("div") - .filter(has_text=re.compile(r"^Select channel to notify$")) - .first - ) - select_channel_placeholder.wait_for() - select_channel_placeholder.click() - - wait_for_loading_finished(page) - - action_channel_input = page.locator( - '[id="triggerDefinitions\\[0\\]\\.actions\\.0\\.destination_id"]' - ) - action_channel_input.wait_for() - action_channel_input.fill(channel_name) - - action_channel_option = page.get_by_text(f"[Channel] {channel_name}").first - action_channel_option.wait_for() - action_channel_option.click() - - create_monitor_button = page.get_by_role("button", name="Create", exact=True) - create_monitor_button.wait_for() - create_monitor_button.click() - - wait_for_loading_finished(page) - - expect(page.get_by_role("heading", name=monitor_name, exact=True)).to_be_visible() - expect(page.get_by_text("Enabled")).to_be_visible() - - -def delete_notifications_channel(page, channel_name): - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect(page.get_by_text(channel_name, exact=True)).to_be_visible() - - select_table_item_checkbox(page, channel_name) - - open_actions_menu(page) - - click_delete_button(page) - - fill_delete_confirm_placeholder(page) - - click_delete_button(page) - - wait_for_channels_header(page) - - expect(page.get_by_text(channel_name, exact=True)).not_to_be_visible() - - -def delete_email_recipient_group(page, recipient_group_name): - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect(page.get_by_text(recipient_group_name, exact=True)).to_be_visible() - - select_table_item_checkbox(page, recipient_group_name) - - delete_recipient_group_button = page.get_by_role( - "button", name="Delete", exact=True - ) - delete_recipient_group_button.wait_for() - delete_recipient_group_button.click() - - fill_delete_confirm_placeholder(page) - - click_delete_button(page) - - wait_for_header(page, re.compile(r"^Email recipient groups$")) - - expect(page.get_by_text(recipient_group_name, exact=True)).not_to_be_visible() - - -def delete_email_smtp_sender(page, email_sender_name): - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect(page.get_by_text(email_sender_name, exact=True)).to_be_visible() - - select_table_item_checkbox(page, email_sender_name) - - delete_email_smtp_sender_button = page.get_by_role( - "button", name="Delete", exact=True - ).first - delete_email_smtp_sender_button.wait_for() - delete_email_smtp_sender_button.click() - - fill_delete_confirm_placeholder(page) - - click_delete_button(page) - - wait_for_header(page, re.compile(r"^Email senders$")) - - expect(page.get_by_text(email_sender_name, exact=True)).not_to_be_visible() - - -def delete_alert_monitor(page, monitor_name): - monitors_loading_message = page.get_by_text("Loading monitors") - monitors_loading_message.wait_for() - expect(monitors_loading_message).not_to_be_visible() - - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect(page.get_by_text(monitor_name, exact=True)).to_be_visible() - - wait_for_loading_finished(page) - - select_table_item_checkbox(page, monitor_name) - - open_actions_menu(page) - - click_delete_button(page) - - click_delete_button(page) - - wait_for_loading_finished(page) - - wait_for_header(page, "Monitors") - - expect(page.get_by_text(monitor_name, exact=True)).not_to_be_visible() - - -def failure_on_edit_save(page, expected_failure_message): - click_save_button(page) - - failure_message = page.get_by_text(expected_failure_message) - expect(failure_message).to_be_visible() - - dismiss_toast_notification_button(page) - - cancel_button = page.get_by_role("link", name="Cancel") - cancel_button.wait_for() - cancel_button.click() diff --git a/e2e/test_alerting_access.py b/e2e/test_alerting_access.py deleted file mode 100644 index dc6fd56..0000000 --- a/e2e/test_alerting_access.py +++ /dev/null @@ -1,418 +0,0 @@ -#!/usr/bin/env python - -import re -import time -from playwright.sync_api import expect -from .notifications import ( - create_email_recipient_group, - create_email_smtp_sender, - create_notifications_channel, - delete_notifications_channel, - delete_email_recipient_group, - create_alert_monitor, - delete_alert_monitor, - delete_email_smtp_sender, - fill_email_recipient_group_details, - failure_on_edit_save, - fill_email_smtp_sender_details, -) -from .utils import ( - log_in, - switch_tenants, - open_primary_menu_link, - click_contextual_menu_link, - click_tab_link, - wait_for_loading_finished, - select_table_item_checkbox, - update_rows_per_table, - click_table_edit_button, - click_actions_edit_link, - click_save_button, - dismiss_toast_notification_button, -) -from . import AUTH_PROXY_URL, CF_ORG_1_NAME, CF_ORG_2_NAME, CF_ORG_3_NAME - -test_run_timestamp = int(time.time()) -test_object_prefix = "E2E-Test" -test_email_recipient_group_name = ( - f"{test_object_prefix}EmailRecipientGroup-{test_run_timestamp}" -) -test_email_smtp_sender_name = ( - f"{test_object_prefix}EmailSmtpSender-{test_run_timestamp}" -).lower() -test_channel_name = f"{test_object_prefix}Channel-{test_run_timestamp}" -test_monitor_name = f"{test_object_prefix}Monitor-{test_run_timestamp}" -test_trigger_name = f"{test_object_prefix}Trigger-{test_run_timestamp}" -test_action_name = f"{test_object_prefix}Action-{test_run_timestamp}" - - -def test_user_can_create_alerts(user_1, page): - def handler(): - dismiss_toast_notification_button(page) - - page.add_locator_handler( - page.get_by_text(re.compile(r"^.*successfully created.$")), - handler, - ) - - log_in(user_1, page, AUTH_PROXY_URL) - - switch_tenants(page, CF_ORG_1_NAME) - - open_primary_menu_link(page, "Notifications") - - click_contextual_menu_link(page, "Email recipient groups") - - create_email_recipient_group(page, user_1, test_email_recipient_group_name) - - click_contextual_menu_link(page, "Email senders") - - create_email_smtp_sender(page, test_email_smtp_sender_name) - - click_contextual_menu_link(page, "Channels") - - create_notifications_channel( - page, - test_email_recipient_group_name, - test_email_smtp_sender_name, - test_channel_name, - ) - - open_primary_menu_link(page, "Alerting") - - click_tab_link(page, "Monitors") - - create_alert_monitor( - page, test_monitor_name, test_trigger_name, test_action_name, test_channel_name - ) - - -def test_user_cannot_see_alert_objects(user_2, page): - log_in(user_2, page, AUTH_PROXY_URL) - - switch_tenants(page, CF_ORG_2_NAME) - - open_primary_menu_link(page, "Notifications") - - click_contextual_menu_link(page, "Email recipient groups") - - wait_for_loading_finished(page) - - expect( - page.get_by_text(test_email_recipient_group_name, exact=True) - ).not_to_be_visible() - - click_contextual_menu_link(page, "Email senders") - - wait_for_loading_finished(page) - - expect( - page.get_by_text(test_email_smtp_sender_name, exact=True) - ).not_to_be_visible() - - click_contextual_menu_link(page, "Channels") - - wait_for_loading_finished(page) - - expect(page.get_by_text(test_channel_name, exact=True)).not_to_be_visible() - - open_primary_menu_link(page, "Alerting") - - click_tab_link(page, "Monitors") - - wait_for_loading_finished(page) - - expect(page.get_by_text(test_monitor_name, exact=True)).not_to_be_visible() - - -def test_user_can_see_but_not_edit_alert_objects(user_3, page): - def handler(): - dismiss_toast_notification_button(page) - - page.add_locator_handler( - page.get_by_text(re.compile(r"^There was a problem loading.*")), - handler, - ) - - log_in(user_3, page, AUTH_PROXY_URL) - - switch_tenants(page, CF_ORG_1_NAME) - - open_primary_menu_link(page, "Notifications") - - click_contextual_menu_link(page, "Email recipient groups") - - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect( - page.get_by_text(test_email_recipient_group_name, exact=True) - ).to_be_visible() - - select_table_item_checkbox(page, test_email_recipient_group_name) - click_table_edit_button(page) - wait_for_loading_finished(page) - - expect(page.get_by_role("heading", name="Edit recipient group")).to_be_visible() - - fill_email_recipient_group_details(page, user_3, test_email_recipient_group_name) - wait_for_loading_finished(page) - failure_on_edit_save(page, "Failed to update recipient group") - - click_contextual_menu_link(page, "Email senders") - wait_for_loading_finished(page) - - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect(page.get_by_text(test_email_smtp_sender_name, exact=True)).to_be_visible() - - select_table_item_checkbox(page, test_email_smtp_sender_name) - click_table_edit_button(page) - wait_for_loading_finished(page) - - expect(page.get_by_role("heading", name="Edit SMTP sender")).to_be_visible() - - fill_email_smtp_sender_details(page, test_email_smtp_sender_name) - wait_for_loading_finished(page) - failure_on_edit_save(page, "Failed to update sender") - - click_contextual_menu_link(page, "Channels") - - update_rows_per_table(page) - wait_for_loading_finished(page) - - channel_link = page.get_by_role("link", name=test_channel_name, exact=True) - expect(channel_link).to_be_visible() - channel_link.click() - - expect(page.get_by_role("heading", name="-")).to_be_visible() - - channel_breadcrumb_link = page.get_by_role("link", name="Channels", exact=True) - channel_breadcrumb_link.wait_for() - channel_breadcrumb_link.click() - - select_table_item_checkbox(page, test_channel_name) - click_actions_edit_link(page) - wait_for_loading_finished(page) - - channel_name_input = page.get_by_label("Name") - channel_name_input.wait_for() - channel_name_input.fill(test_channel_name) - - slack_webhook_input = page.get_by_label("Slack webhook URL") - slack_webhook_input.wait_for() - slack_webhook_input.fill("https://hooks.slack.com/services/foo/bar") - - wait_for_loading_finished(page) - failure_on_edit_save(page, "Failed to update channel") - - open_primary_menu_link(page, "Alerting") - click_tab_link(page, "Monitors") - - update_rows_per_table(page) - wait_for_loading_finished(page) - - monitor_link = page.get_by_text(test_monitor_name, exact=True) - expect(monitor_link).to_be_visible() - - monitor_link_href = re.sub(r"\?.*$", "", monitor_link.get_attribute("href")) - - monitor_link.click() - - page.wait_for_url(f"**/app/alerting{monitor_link_href}*") - page.wait_for_url(re.compile(r".*/app/alerting#/monitors\?.*")) - - # should redirect back to main monitors page - expect(page).to_have_url(re.compile(r"app/alerting#/monitors\?.*")) - - select_table_item_checkbox(page, test_monitor_name) - click_actions_edit_link(page) - - page.wait_for_url(f"**/app/alerting{monitor_link_href}*") - page.wait_for_url(re.compile(r".*/app/alerting#/monitors\?.*")) - - # should redirect back to main monitors page - expect(page).to_have_url(re.compile(r"app/alerting#/monitors\?.*")) - - -def test_user_can_see_and_edit_alert_objects(user_4, page): - def handler(): - dismiss_toast_notification_button(page) - - page.add_locator_handler( - page.get_by_text(re.compile(r"^.*successfully updated.$")), - handler, - ) - - log_in(user_4, page, AUTH_PROXY_URL) - - # using this tenant should not affect access to alerting objects - switch_tenants(page, CF_ORG_3_NAME) - - open_primary_menu_link(page, "Notifications") - - # Email recipient groups - - # Verify we can view and edit the email recipient group - click_contextual_menu_link(page, "Email recipient groups") - - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect( - page.get_by_text(test_email_recipient_group_name, exact=True) - ).to_be_visible() - - select_table_item_checkbox(page, test_email_recipient_group_name) - click_table_edit_button(page) - wait_for_loading_finished(page) - - expect(page.get_by_role("heading", name="Edit recipient group")).to_be_visible() - - click_save_button(page) - wait_for_loading_finished(page) - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect( - page.get_by_role("heading", name="Email recipient groups", exact=True) - ).to_be_visible() - expect( - page.get_by_text(test_email_recipient_group_name, exact=True) - ).to_be_visible() - - # Email senders - click_contextual_menu_link(page, "Email senders") - wait_for_loading_finished(page) - - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect(page.get_by_text(test_email_smtp_sender_name, exact=True)).to_be_visible() - - select_table_item_checkbox(page, test_email_smtp_sender_name) - click_table_edit_button(page) - wait_for_loading_finished(page) - - expect(page.get_by_role("heading", name="Edit SMTP sender")).to_be_visible() - - click_save_button(page) - wait_for_loading_finished(page) - update_rows_per_table(page) - wait_for_loading_finished(page) - - expect( - page.get_by_role("heading", name="Email senders", exact=True) - ).to_be_visible() - expect(page.get_by_text(test_email_smtp_sender_name, exact=True)).to_be_visible() - - # Channels - - # Verify we can view and edit the notification channel - click_contextual_menu_link(page, "Channels") - - update_rows_per_table(page) - wait_for_loading_finished(page) - - channel_link = page.get_by_role("link", name=test_channel_name, exact=True) - expect(channel_link).to_be_visible() - channel_link.click() - - expect( - page.get_by_role("heading", name=test_channel_name, exact=True) - ).to_be_visible() - - click_actions_edit_link(page) - wait_for_loading_finished(page) - - expect(page.get_by_role("heading", name="Edit channel")).to_be_visible() - - click_save_button(page) - wait_for_loading_finished(page) - - expect( - page.get_by_role("heading", name=test_channel_name, exact=True) - ).to_be_visible() - - # Verify we can view and edit the alert monitor - open_primary_menu_link(page, "Alerting") - - click_tab_link(page, "Monitors") - - monitors_loading_message = page.get_by_text("Loading monitors") - monitors_loading_message.wait_for() - expect(monitors_loading_message).not_to_be_visible() - - update_rows_per_table(page) - wait_for_loading_finished(page) - - monitor_link = page.get_by_text(test_monitor_name, exact=True) - expect(monitor_link).to_be_visible() - monitor_link.click() - - wait_for_loading_finished(page) - - expect( - page.get_by_role("heading", name=test_monitor_name, exact=True) - ).to_be_visible() - - wait_for_loading_finished(page) - - monitor_edit_button = page.get_by_role("button", name="Edit", exact=True) - expect(monitor_edit_button).to_be_visible() - expect(monitor_edit_button).to_be_enabled() - monitor_edit_button.click() - - wait_for_loading_finished(page) - - time_field_input = page.get_by_text("@timestamp").first - time_field_input.wait_for() - time_field_input.click() - - timestamp_option_button = page.get_by_role("option", name="@timestamp", exact=True) - expect(timestamp_option_button).to_be_visible() - timestamp_option_button.click() - - click_save_button(page) - wait_for_loading_finished(page) - - expect( - page.get_by_role("heading", name=test_monitor_name, exact=True) - ).to_be_visible() - - -def test_user_can_delete_alerts(user_1, page): - def handler(): - dismiss_toast_notification_button(page) - - page.add_locator_handler( - page.get_by_text( - re.compile(r"^.*(successfully deleted|deleted successfully).$") - ), - handler, - ) - - log_in(user_1, page, AUTH_PROXY_URL) - - switch_tenants(page, CF_ORG_1_NAME) - - open_primary_menu_link(page, "Alerting") - - click_tab_link(page, "Monitors") - - delete_alert_monitor(page, test_monitor_name) - - open_primary_menu_link(page, "Notifications") - - click_contextual_menu_link(page, "Channels") - - delete_notifications_channel(page, test_channel_name) - - click_contextual_menu_link(page, "Email recipient groups") - - delete_email_recipient_group(page, test_email_recipient_group_name) - - click_contextual_menu_link(page, "Email senders") - - delete_email_smtp_sender(page, test_email_smtp_sender_name) diff --git a/e2e/test_smoke_tests_login.py b/e2e/test_smoke_tests_login.py deleted file mode 100644 index b05d7f4..0000000 --- a/e2e/test_smoke_tests_login.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python - -import re -import logging -from playwright.sync_api import expect -from urllib.parse import urljoin - -from .utils import log_in -from . import AUTH_PROXY_URL - -logging.basicConfig(level=logging.DEBUG) - - -def test_user_login(user_1, page): - log_in(user_1, page, AUTH_PROXY_URL) - expect(page).to_have_url(re.compile(f"{urljoin(AUTH_PROXY_URL, 'app/home')}.*")) diff --git a/e2e/user.py b/e2e/user.py deleted file mode 100644 index d504129..0000000 --- a/e2e/user.py +++ /dev/null @@ -1,9 +0,0 @@ -import pyotp - - -class User: - def __init__(self, username, password, totp_seed): - self.username = username - self.password = password - self.totp_seed = totp_seed - self.totp = pyotp.TOTP(totp_seed) diff --git a/e2e/utils.py b/e2e/utils.py deleted file mode 100644 index 519cffd..0000000 --- a/e2e/utils.py +++ /dev/null @@ -1,179 +0,0 @@ -import re -from playwright.sync_api import expect - -from . import AUTH_PROXY_URL, UAA_BASE_URL - - -def log_in(user, page, start_at): - # go to auth proxy - page.goto(start_at) - - # accept the monitoring notice - agree_continue_button = page.get_by_text("AGREE AND CONTINUE") - agree_continue_button.wait_for() - agree_continue_button.click() - - # select the cloud.gov IdP - cloud_gov_idp_button = page.get_by_role("link", name="cloud.gov") - cloud_gov_idp_button.wait_for() - cloud_gov_idp_button.click() - - username_field = page.get_by_label("Email address") - password_field = page.get_by_label("Password") - username_field.wait_for() - password_field.wait_for() - username_field.fill(user.username) - password_field.fill(user.password) - - login_button = page.get_by_text("Login") - login_button.wait_for() - login_button.click() - - totp_field = page.locator("css=input[id='j_tokenNumber']") - totp_field.wait_for() - totp_field.fill(user.totp.now()) - - login_button = page.get_by_text("Login") - login_button.wait_for() - login_button.click() - - # wait for OAuth authorize page or auth proxy page - page.wait_for_url(re.compile(f"({AUTH_PROXY_URL}|{UAA_BASE_URL})")) - - # if OAuth authorize page, then authorize the application - if "/authorize?" in page.url: - # first time using this app with this user - authorize_button = page.get_by_text("Authorize") - authorize_button.wait_for() - authorize_button.click() - - -def switch_tenants(page, tenant="Global"): - """ - switch to the specified tenant. - """ - - select_tenant_header = page.get_by_role( - "heading", name="Select your tenant", exact=True - ) - select_tenant_header.wait_for() - - custom_tenant_checkbox = page.get_by_label("Choose from custom") - expect(custom_tenant_checkbox).to_be_visible() - expect(custom_tenant_checkbox).to_be_enabled() - - selected_tenant = page.get_by_role("combobox").get_by_text(tenant, exact=True) - - if not selected_tenant.is_visible(): - open_tenant_options_btn = page.get_by_label("Open list of options") - open_tenant_options_btn.wait_for() - open_tenant_options_btn.click() - - tenant_option = page.get_by_role("option", name=tenant) - tenant_option.wait_for() - tenant_option.click() - - # submit - submit_button = page.get_by_text("Confirm") - submit_button.wait_for() - submit_button.click() - - # wait for loading screen - loading_text = page.get_by_text("Loading Cloud.gov Logs") - loading_text.wait_for() - - # wait for dashboard to finish loading - dashboards_title = page.get_by_role("heading", name="Dashboards") - dashboards_title.wait_for() - - -def open_primary_menu_link(page, menu_link_name): - # open the hamburger menu - hamburger_button = page.get_by_label("Toggle primary navigation") - hamburger_button.wait_for() - hamburger_button.click() - - menu_link = page.get_by_text(menu_link_name) - menu_link.wait_for() - menu_link.click() - - -def click_contextual_menu_link(page, link_name): - link = page.locator("#app-wrapper").get_by_role("link", name=link_name, exact=True) - link.wait_for() - link.click() - - -def wait_for_header(page, header_name): - channels_header = page.get_by_role("heading", name=header_name) - channels_header.wait_for() - - -def fill_delete_confirm_placeholder(page): - delete_confirm_input = page.get_by_placeholder("delete") - delete_confirm_input.wait_for() - delete_confirm_input.fill("delete") - - -def open_actions_menu(page): - actions_button = page.get_by_role("button", name="Actions", exact=True) - actions_button.wait_for() - actions_button.click() - - -def click_actions_edit_link(page): - open_actions_menu(page) - actions_edit_button = page.get_by_role("button", name="Edit", exact=True) - actions_edit_button.wait_for() - actions_edit_button.click() - - -def click_delete_button(page): - delete_button = page.get_by_role("button", name="Delete", exact=True) - delete_button.wait_for() - delete_button.click() - - -def select_table_item_checkbox(page, item_text): - checkbox = page.locator("tr").filter(has_text=item_text).get_by_role("checkbox") - checkbox.wait_for() - checkbox.click() - - -def click_table_edit_button(page): - edit_button = page.get_by_role("button", name="Edit", exact=True).first - expect(edit_button).to_be_visible() - expect(edit_button).to_be_enabled() - edit_button.click() - - -def click_tab_link(page, link_text): - link = page.get_by_role("tab", name=link_text) - link.wait_for() - link.click() - - -def wait_for_loading_finished(page): - expect(page.get_by_label("Loading content")).not_to_be_visible() - - -def update_rows_per_table(page, rows_option="50 rows"): - rows_per_page_button = page.get_by_role( - "button", name=re.compile(r"^Rows per page: [0-9]+$") - ) - rows_per_page_button.wait_for() - rows_per_page_button.click() - - fifty_rows_button = page.get_by_role("button", name=rows_option, exact=True) - fifty_rows_button.wait_for() - fifty_rows_button.click() - - -def dismiss_toast_notification_button(page): - page.get_by_label("Dismiss toast").click() - - -def click_save_button(page): - save_button = page.get_by_role("button", name="Save") - save_button.wait_for() - save_button.click() diff --git a/opsfiles/enable-auth-proxy-route-dev.yml b/opsfiles/enable-auth-proxy-route-dev.yml deleted file mode 100644 index 897857e..0000000 --- a/opsfiles/enable-auth-proxy-route-dev.yml +++ /dev/null @@ -1,11 +0,0 @@ -# add nats info for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/nats?/tls? - value: - client_cert: ((/bosh/cf-development/nats_client_cert.certificate)) - client_key: ((/bosh/cf-development/nats_client_cert.private_key)) - enabled: true - -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/consumes?/nats-tls?/deployment? - value: cf-development diff --git a/opsfiles/enable-auth-proxy-route-production.yml b/opsfiles/enable-auth-proxy-route-production.yml deleted file mode 100644 index 549f05d..0000000 --- a/opsfiles/enable-auth-proxy-route-production.yml +++ /dev/null @@ -1,16 +0,0 @@ -# add nats info for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/nats?/tls? - value: - client_cert: ((/bosh/cf-production/nats_client_cert.certificate)) - client_key: ((/bosh/cf-production/nats_client_cert.private_key)) - enabled: true - -# add address for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/route_registrar?/routes?/name=opensearch-auth-proxy/uris?/- - value: logs.fr.cloud.gov - -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/consumes?/nats-tls?/deployment? - value: cf-production diff --git a/opsfiles/enable-auth-proxy-route-staging.yml b/opsfiles/enable-auth-proxy-route-staging.yml deleted file mode 100644 index 4a0a542..0000000 --- a/opsfiles/enable-auth-proxy-route-staging.yml +++ /dev/null @@ -1,16 +0,0 @@ -# add nats info for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/nats?/tls? - value: - client_cert: ((/bosh/cf-staging/nats_client_cert.certificate)) - client_key: ((/bosh/cf-staging/nats_client_cert.private_key)) - enabled: true - -# add address for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/route_registrar?/routes?/name=opensearch-auth-proxy/uris?/- - value: logs.fr-stage.cloud.gov - -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/consumes?/nats-tls?/deployment? - value: cf-staging diff --git a/opsfiles/enable-auth-proxy-route-test.yml b/opsfiles/enable-auth-proxy-route-test.yml deleted file mode 100644 index 0970673..0000000 --- a/opsfiles/enable-auth-proxy-route-test.yml +++ /dev/null @@ -1,16 +0,0 @@ -# add nats info for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/nats?/tls? - value: - client_cert: ((/bosh/cf-development/nats_client_cert.certificate)) - client_key: ((/bosh/cf-development/nats_client_cert.private_key)) - enabled: true - -# add address for auth proxy route -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/properties?/route_registrar?/routes?/name=opensearch-auth-proxy/uris?/- - value: logs-test.dev.us-gov-west-1.aws-us-gov.cloud.gov - -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar/consumes?/nats-tls?/deployment? - value: cf-development diff --git a/opsfiles/enable-proxy-auth.yml b/opsfiles/enable-proxy-auth.yml deleted file mode 100644 index b02511a..0000000 --- a/opsfiles/enable-proxy-auth.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- - -# opensearch_manager -- type: replace - path: /instance_groups/name=opensearch_manager/jobs/name=opensearch/properties/opensearch?/enable_proxy_auth - value: true - -# maintenance -- type: replace - path: /instance_groups/name=maintenance/jobs/name=opensearch/properties/opensearch?/enable_proxy_auth - value: true - -# opensearch_data -- type: replace - path: /instance_groups/name=opensearch_data/jobs/name=opensearch/properties/opensearch?/enable_proxy_auth - value: true - -# opensearch_old_data -- type: replace - path: /instance_groups/name=opensearch_old_data/jobs/name=opensearch/properties/opensearch?/enable_proxy_auth - value: true - -# opensearch_dashboards -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth - value: true - -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch_dashboards/properties?/opensearch_dashboards?/opensearch?/enable_proxy_auth - value: true - -# ingestor -- type: replace - path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/enable_proxy_auth - value: true - -# add variable for auth proxy certs -- type: replace - path: /variables/name=auth_proxy? - value: - name: auth_proxy - options: - alternative_names: - - localhost - - auth-proxy.opensearch.internal - ca: opensearch_ca - common_name: auth-proxy.opensearch.internal - extended_key_usage: - - server_auth - - client_auth - type: certificate - update_mode: converge - -# add auth proxy job -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=opensearch-dashboards-cf-auth-proxy? - value: - consumes: - opensearch: - from: opensearch_manager - ip_addresses: true - name: opensearch-dashboards-cf-auth-proxy - properties: - opensearch-dashboards-cf-auth-proxy: - cf: - admin_group: ((cf-admin-group)) - api_url: ((cf-api-url)) - auditor_group: ((cf-auditor-group)) - gunicorn: - num_workers: 4 - timeout: 300 - ssl: - ca: ((opensearch_node.ca)) - certificate: ((auth_proxy.certificate)) - key: ((auth_proxy.private_key)) - opensearch_dashboards: - ssl: - ca: /var/vcap/jobs/opensearch_dashboards/config/ssl/opensearch.ca - certificate: /var/vcap/jobs/opensearch_dashboards/config/ssl/dashboard-web.crt - key: /var/vcap/jobs/opensearch_dashboards/config/ssl/dashboard-web.key - url: https://localhost:5605 - port: 8000 - secret_key: ((auth-proxy-secret-key)) - uaa: - auth_url: ((uaa-auth-url)) - base_url: ((uaa-base-url)) - client: - id: ((uaa-client-id)) - secret: ((uaa-client-secret)) - jwks: ((uaa-jwks)) - redis: - host: ((redis_host)) - password: ((redis_password)) - release: opensearch - -# add route for auth proxy job -- type: replace - path: /instance_groups/name=opensearch_dashboards/jobs/name=route_registrar? - value: - consumes: - nats-tls: - from: nats-tls - name: route_registrar - properties: - route_registrar: - routes: - - name: opensearch-auth-proxy - registration_interval: 2s - server_cert_domain_san: auth-proxy.opensearch.internal - timeout: 1s - tls_port: 8000 - release: routing - -# variables -- type: replace - path: /variables/name=auth-proxy-secret-key? - value: - name: auth-proxy-secret-key - type: password \ No newline at end of file diff --git a/scripts/download-e2e-ci-results.sh b/scripts/download-e2e-ci-results.sh deleted file mode 100755 index 7f17ea3..0000000 --- a/scripts/download-e2e-ci-results.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -BUILD_NUMBER=$1 - -if [[ -z "$BUILD_NUMBER" ]]; then - echo "build number is required as first argument" - exit 1 -fi - -ENVIRONMENT=${2:-production} - -CI_TASK_TARGET="fly -t ${FLY_TARGET:=ci} intercept -j deploy-logs-platform/e2e-tests-$ENVIRONMENT -s e2e-tests -b $BUILD_NUMBER" -TEST_RESULTS_DIR="deploy-logs-platform-config/test-results" -LOCAL_TARGET_DIR="ci-test-results" - -for test_dir in $($CI_TASK_TARGET -- ls $TEST_RESULTS_DIR); do - echo "found test dir: $test_dir" - - for file in $($CI_TASK_TARGET -- ls "$TEST_RESULTS_DIR/$test_dir"); do - mkdir -p "$LOCAL_TARGET_DIR/$test_dir" - $CI_TASK_TARGET -- cat "$TEST_RESULTS_DIR/$test_dir/$file" > "$LOCAL_TARGET_DIR/$test_dir/$file" - echo "downloaded $file" - done -done diff --git a/scripts/e2e-local.sh b/scripts/e2e-local.sh deleted file mode 100755 index 26177c0..0000000 --- a/scripts/e2e-local.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -x - -set -o allexport -source ".env" -set +o allexport - -dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -"$dir"/e2e.sh --headed "$@" diff --git a/scripts/e2e.sh b/scripts/e2e.sh deleted file mode 100755 index 1a34c85..0000000 --- a/scripts/e2e.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -python -m venv venv -source venv/bin/activate -pip install -r requirements-test.txt -playwright install firefox -python -m pytest e2e --browser firefox --tracing retain-on-failure "$@" diff --git a/varsfiles/logsearch_ingestor b/varsfiles/logsearch_ingestor new file mode 100644 index 0000000..754e084 --- /dev/null +++ b/varsfiles/logsearch_ingestor @@ -0,0 +1,83 @@ +- name: ingestor + instances: 5 + jobs: + - name: bpm + release: bpm + - name: elasticsearch-platform + release: logsearch + consumes: + elasticsearch-platform: {from: elasticsearch_master} + properties: + elasticsearch: + jvm_options: + - "-Dlog4j2.formatMsgNoLookups=true" + node: + roles: ingest + heap_size: 1G + config_options: {"xpack.monitoring.enabled": false} + - name: ingestor_syslog + release: logsearch + consumes: + elasticsearch: {from: elasticsearch_master} + provides: + ingestor: {as: ingestor_link} + properties: + logstash: + jvm_options: + - "-Dlog4j2.formatMsgNoLookups=true" + queue: + max_bytes: 30gb + logstash_parser: + elasticsearch: + # Use per-day indexing strategy + index: "logs-platform-%{+YYYY.MM.dd}" + index_type: "%{@type}" + data_hosts: [127.0.0.1] + filters: + - path: /var/vcap/packages/logsearch-config-logstash-filters/logstash-filters-default.conf + - content: | + if [@source][component] == "snort" { + grok { + match => { + "@message" => "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:msg}\s\{%{DATA:proto}\}\s%{IP:src_ip}:%{INT:src_port}\s->\s%{IP:dst_ip}:%{INT:dst_port}" + } + } + } + if [@source][component] == "clamd" { + grok { + match => { + "@message" => "%{WORD:event_type}:\s%{DATA:file_path}:\s%{DATA:signature_name}\sFOUND" + } + } + } + outputs: + - plugin: s3 + options: + region: ((region)) + bucket: ((bucket)) + access_key_id: ((access_key_id)) + secret_access_key: ((secret_access_key)) + server_side_encryption: true + prefix: "%{+yyyy/MM/dd/HH/mm}" + encoding: "gzip" + temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp + # note that this is different than in the archivers + # here, we're uploading the parsed message as json, not the raw log line + codec: json + - plugin: elasticsearch + options: {} + deployment_dictionary: + - /var/vcap/packages/logsearch-config/deployment_lookup.yml + - /var/vcap/jobs/parser-config-lfc/config/deployment_lookup.yml + - name: parser-config-lfc + release: logsearch-for-cloudfoundry + vm_type: logsearch_ingestor + vm_extensions: + - platform-syslog-lb + - 10GB_root_disk + - 10GB_ephemeral_disk + persistent_disk_type: logsearch_ingestor + stemcell: default + azs: [z1, z2] + networks: + - name: services \ No newline at end of file diff --git a/varsfiles/opensearach_ingestor b/varsfiles/opensearach_ingestor new file mode 100644 index 0000000..4b6a18c --- /dev/null +++ b/varsfiles/opensearach_ingestor @@ -0,0 +1,93 @@ +- name: ingestor + instances: 1 + jobs: + - name: bpm + release: bpm + - name: opensearch + release: opensearch + consumes: *consumes-opensearch-manager + properties: + opensearch: + heap_size: 1G + http_host: 127.0.0.1 + jvm_options: + - -Dlog4j2.formatMsgNoLookups=true + oidc: + enabled: true + hostname: ((oidc_host)) + client_id: ((client_id)) + client_secret: ((client_secret)) + - name: ingestor_syslog + consumes: *consumes-opensearch-manager + properties: + logstash: + jvm_options: + - -Dlog4j2.formatMsgNoLookups=true + queue: + max_bytes: 30gb + logstash_parser: + opensearch: + data_hosts: + - localhost + index: ((alias)) + index_type: '%{@type}' + ssl: + ca: ((opensearch_node.ca)) + certificate: ((logstash.certificate)) + private_key: ((logstash.private_key)) + filters: + - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-filters-default.conf + - content: | + if [@source][component] == "snort" { + grok { + match => { + "@message" => "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:msg}\s\{%{DATA:proto}\}\s%{IP:src_ip}:%{INT:src_port}\s->\s%{IP:dst_ip}:%{INT:dst_port}" + } + } + } + if [@source][component] == "clamd" { + grok { + match => { + "@message" => "%{WORD:event_type}:\s%{DATA:file_path}:\s%{DATA:signature_name}\sFOUND" + } + } + } + outputs: + - plugin: s3 + options: + region: ((region)) + bucket: ((bucket)) + access_key_id: ((access_key_id)) + secret_access_key: ((secret_access_key)) + server_side_encryption: true + prefix: "%{+yyyy/MM/dd/HH/mm}" + encoding: "gzip" + temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp + # note that this is different than in the archivers + # here, we're uploading the parsed message as json, not the raw log line + codec: json + - plugin: opensearch + options: {} + deployment_dictionary: + - /var/vcap/packages/base-logstash-filters/deployment_lookup.yml + - /var/vcap/jobs/deployment_lookup_config/config/deployment_lookup.yml + logstash_ingestor: + syslog: + port: 5514 + provides: + ingestor: + as: ingestor_link + release: opensearch + - name: deployment_lookup_config + release: opensearch + vm_type: t3.medium + vm_extensions: + - platform-syslog-lb + - 20GB_ephemeral_disk + persistent_disk_type: logs_opensearch_ingestor + stemcell: default + azs: [z1,z2] + networks: + - name: services + update: + serial: true # Block on this job to create deploy group 5 \ No newline at end of file From 0f21d059a421947286b6e5fd24848224cdf11367 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 28 Apr 2026 16:18:15 -0400 Subject: [PATCH 136/165] catching up with staging --- opensearch-scaling-staging.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml index d6c76b2..06456b8 100644 --- a/opensearch-scaling-staging.yml +++ b/opensearch-scaling-staging.yml @@ -4,7 +4,7 @@ - type: replace path: /instance_groups/name=opensearch_manager?/instances? - value: 1 + value: 3 - type: replace @@ -57,7 +57,7 @@ - type: replace path: /instance_groups/name=ingestor?/instances? - value: 1 + value: 5 - type: replace path: /instance_groups/name=ingestor?/vm_type? From 6cef2f2b1f0abb2ace60a8626df72e982fcfc965 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 29 Apr 2026 14:45:42 -0400 Subject: [PATCH 137/165] adding in production manigest --- ci/pipeline.yml | 75 +++++++++++++++++++ opensearch-scaling-production.yml | 119 ++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 opensearch-scaling-production.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 2181d97..17a0aa9 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -9,6 +9,11 @@ groups: - name: staging jobs: - deploy-opensearch-staging + - upload-dashboards-objects-staging + - smoke-tests-staging + - name: production + jobs: + - deploy-opensearch-production jobs: - name: build-opensearch-test-release @@ -254,6 +259,60 @@ jobs: BOSH_FLAGS: "--keep-alive" BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) + - name: deploy-opensearch-production + serial_groups: [bosh-production] + plan: + - in_parallel: + - get: pipeline-tasks + - get: deploy-logs-platform-config + passed: [smoke-tests-staging] + trigger: true + - get: opensearch-test-release + passed: [smoke-tests-staging] + trigger: true + - get: opensearch-stemcell-jammy + passed: [smoke-tests-staging] + trigger: true + - get: general-task + - get: terraform-yaml + resource: terraform-yaml-production + trigger: true + - task: opensearch-manifest + image: general-task + config: + platform: linux + inputs: + - name: deploy-logs-platform-config + run: + path: sh + args: + - -exc + - | + bosh int deploy-logs-platform-config/opensearch-base.yml \ + -o deploy-logs-platform-config/opensearch-scaling-production.yml \ + > opensearch-manifest/manifest.yml + outputs: + - name: opensearch-manifest + - task: terraform-secrets + image: general-task + file: deploy-logs-platform-config/ci/terraform-secrets.yml + - put: opensearch-production-deployment + params: + manifest: opensearch-manifest/manifest.yml + releases: + - opensearch-test-release/*.tgz + stemcells: + - opensearch-stemcell-jammy/*.tgz + ops_files: + - deploy-logs-platform-config/opsfiles/enable-node-tls.yml + - deploy-logs-platform-config/opsfiles/enable-dashboard-dns.yml + - deploy-logs-platform-config/opsfiles/enable-dashboards-tls.yml + - deploy-logs-platform-config/opsfiles/enable-oidc-tls.yml + - deploy-logs-platform-config/opsfiles/enable-syslog.yml + - deploy-logs-platform-config/opsfiles/opensearch-notification.yml + - deploy-logs-platform-config/opsfiles/add-keystore-credentials-errand.yml + vars_files: + - terraform-secrets/terraform.yml resources: - name: opensearch-test-release-git-repo @@ -299,6 +358,13 @@ resources: versioned_file: ((tf-state-file-staging)) region_name: ((aws-region)) + - name: terraform-yaml-production + type: s3-iam + source: + bucket: ((tf-state-bucket-production)) + versioned_file: ((tf-state-file-production)) + region_name: ((aws-region)) + - name: tests-timer type: time source: @@ -344,6 +410,15 @@ resources: ca_cert: ((bosh-director-info.staging.ca_cert)) deployment: logs-platform + - name: opensearch-production-deployment + type: bosh-deployment + source: &bosh-params-staging + target: ((bosh-director-info.production.environment)) + client: ((bosh-director-info.production.client)) + client_secret: ((bosh-director-info.production.client_secret)) + ca_cert: ((bosh-director-info.production.ca_cert)) + deployment: logs-platform + - name: general-task type: registry-image source: diff --git a/opensearch-scaling-production.yml b/opensearch-scaling-production.yml new file mode 100644 index 0000000..9dcd8dd --- /dev/null +++ b/opensearch-scaling-production.yml @@ -0,0 +1,119 @@ +- type: replace + path: /instance_groups/name=opensearch_manager?/vm_type? + value: t3.large + +- type: replace + path: /instance_groups/name=opensearch_manager?/instances? + value: 3 + + +- type: replace + path: /instance_groups/name=opensearch_data?/instances? + value: 0 + +- type: replace + path: /instance_groups/name=opensearch_data?/vm_type? + value: r6i.xlarge + +- type: replace + path: /instance_groups/name=opensearch_data?/update? + value: + max_in_flight: 1 + canaries: 1 + serial: true + + +- type: replace + path: /instance_groups/name=opensearch_old_data?/instances? + value: 0 + +- type: replace + path: /instance_groups/name=opensearch_old_data?/vm_type? + value: r6i.xlarge + +- type: replace + path: /instance_groups/name=opensearch_old_data?/update? + value: + max_in_flight: 1 + canaries: 1 + serial: true + + +- type: replace + path: /instance_groups/name=opensearch_dashboards?/instances? + value: 1 + +- type: replace + path: /instance_groups/name=opensearch_dashboards?/vm_type? + value: t3.xlarge + +- type: replace + path: /instance_groups/name=opensearch_dashboards/update? + value: + max_in_flight: 100% + canaries: 100% + serial: true + + +- type: replace + path: /instance_groups/name=ingestor?/instances? + value: 0 + +- type: replace + path: /instance_groups/name=ingestor?/vm_type? + value: m6i.large + + + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? + value: + index_pattern: logs-platform-* + minimum: 1000 + run: true + time_field: '@timestamp' + time_interval: 5m + + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_document_size? + value: 3gb + + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/rollover_index_age? + value: 1d + +- type: replace + path: /instance_groups/name=maintenance?/jobs?/name=upload_opensearch_config?/properties?/opensearch_config?/platform_shard_count? + value: 2 + +- type: replace + path: /instance_groups/name=maintenance?/instances? + value: 1 + +- type: replace + path: /instance_groups/name=maintenance?/vm_type? + value: t3.large + + + + +- type: replace + path: /addons? + value: + - name: bosh-dns-aliases + jobs: + - name: bosh-dns-aliases + release: bosh-dns-aliases + properties: + aliases: + - domain: nats.service.cf.internal + targets: + - deployment: cf-development + domain: bosh + instance_group: nats + network: default + query: '*' + + From 4ab831aa957ace013306064890c15ad1cd9fa458 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 30 Apr 2026 11:27:18 -0400 Subject: [PATCH 138/165] removing these --- varsfiles/logsearch_ingestor | 83 ------------------------------ varsfiles/opensearach_ingestor | 93 ---------------------------------- 2 files changed, 176 deletions(-) delete mode 100644 varsfiles/logsearch_ingestor delete mode 100644 varsfiles/opensearach_ingestor diff --git a/varsfiles/logsearch_ingestor b/varsfiles/logsearch_ingestor deleted file mode 100644 index 754e084..0000000 --- a/varsfiles/logsearch_ingestor +++ /dev/null @@ -1,83 +0,0 @@ -- name: ingestor - instances: 5 - jobs: - - name: bpm - release: bpm - - name: elasticsearch-platform - release: logsearch - consumes: - elasticsearch-platform: {from: elasticsearch_master} - properties: - elasticsearch: - jvm_options: - - "-Dlog4j2.formatMsgNoLookups=true" - node: - roles: ingest - heap_size: 1G - config_options: {"xpack.monitoring.enabled": false} - - name: ingestor_syslog - release: logsearch - consumes: - elasticsearch: {from: elasticsearch_master} - provides: - ingestor: {as: ingestor_link} - properties: - logstash: - jvm_options: - - "-Dlog4j2.formatMsgNoLookups=true" - queue: - max_bytes: 30gb - logstash_parser: - elasticsearch: - # Use per-day indexing strategy - index: "logs-platform-%{+YYYY.MM.dd}" - index_type: "%{@type}" - data_hosts: [127.0.0.1] - filters: - - path: /var/vcap/packages/logsearch-config-logstash-filters/logstash-filters-default.conf - - content: | - if [@source][component] == "snort" { - grok { - match => { - "@message" => "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:msg}\s\{%{DATA:proto}\}\s%{IP:src_ip}:%{INT:src_port}\s->\s%{IP:dst_ip}:%{INT:dst_port}" - } - } - } - if [@source][component] == "clamd" { - grok { - match => { - "@message" => "%{WORD:event_type}:\s%{DATA:file_path}:\s%{DATA:signature_name}\sFOUND" - } - } - } - outputs: - - plugin: s3 - options: - region: ((region)) - bucket: ((bucket)) - access_key_id: ((access_key_id)) - secret_access_key: ((secret_access_key)) - server_side_encryption: true - prefix: "%{+yyyy/MM/dd/HH/mm}" - encoding: "gzip" - temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp - # note that this is different than in the archivers - # here, we're uploading the parsed message as json, not the raw log line - codec: json - - plugin: elasticsearch - options: {} - deployment_dictionary: - - /var/vcap/packages/logsearch-config/deployment_lookup.yml - - /var/vcap/jobs/parser-config-lfc/config/deployment_lookup.yml - - name: parser-config-lfc - release: logsearch-for-cloudfoundry - vm_type: logsearch_ingestor - vm_extensions: - - platform-syslog-lb - - 10GB_root_disk - - 10GB_ephemeral_disk - persistent_disk_type: logsearch_ingestor - stemcell: default - azs: [z1, z2] - networks: - - name: services \ No newline at end of file diff --git a/varsfiles/opensearach_ingestor b/varsfiles/opensearach_ingestor deleted file mode 100644 index 4b6a18c..0000000 --- a/varsfiles/opensearach_ingestor +++ /dev/null @@ -1,93 +0,0 @@ -- name: ingestor - instances: 1 - jobs: - - name: bpm - release: bpm - - name: opensearch - release: opensearch - consumes: *consumes-opensearch-manager - properties: - opensearch: - heap_size: 1G - http_host: 127.0.0.1 - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - oidc: - enabled: true - hostname: ((oidc_host)) - client_id: ((client_id)) - client_secret: ((client_secret)) - - name: ingestor_syslog - consumes: *consumes-opensearch-manager - properties: - logstash: - jvm_options: - - -Dlog4j2.formatMsgNoLookups=true - queue: - max_bytes: 30gb - logstash_parser: - opensearch: - data_hosts: - - localhost - index: ((alias)) - index_type: '%{@type}' - ssl: - ca: ((opensearch_node.ca)) - certificate: ((logstash.certificate)) - private_key: ((logstash.private_key)) - filters: - - logs-for-cf: /var/vcap/packages/cf-logstash-filters/logstash-filters-default.conf - - content: | - if [@source][component] == "snort" { - grok { - match => { - "@message" => "\[%{INT:gid}:%{INT:sid}:%{INT:rev}\]\s%{DATA:msg}\s\{%{DATA:proto}\}\s%{IP:src_ip}:%{INT:src_port}\s->\s%{IP:dst_ip}:%{INT:dst_port}" - } - } - } - if [@source][component] == "clamd" { - grok { - match => { - "@message" => "%{WORD:event_type}:\s%{DATA:file_path}:\s%{DATA:signature_name}\sFOUND" - } - } - } - outputs: - - plugin: s3 - options: - region: ((region)) - bucket: ((bucket)) - access_key_id: ((access_key_id)) - secret_access_key: ((secret_access_key)) - server_side_encryption: true - prefix: "%{+yyyy/MM/dd/HH/mm}" - encoding: "gzip" - temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp - # note that this is different than in the archivers - # here, we're uploading the parsed message as json, not the raw log line - codec: json - - plugin: opensearch - options: {} - deployment_dictionary: - - /var/vcap/packages/base-logstash-filters/deployment_lookup.yml - - /var/vcap/jobs/deployment_lookup_config/config/deployment_lookup.yml - logstash_ingestor: - syslog: - port: 5514 - provides: - ingestor: - as: ingestor_link - release: opensearch - - name: deployment_lookup_config - release: opensearch - vm_type: t3.medium - vm_extensions: - - platform-syslog-lb - - 20GB_ephemeral_disk - persistent_disk_type: logs_opensearch_ingestor - stemcell: default - azs: [z1,z2] - networks: - - name: services - update: - serial: true # Block on this job to create deploy group 5 \ No newline at end of file From d034fb65a9413061497d5ae345dec1b65297884f Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 30 Apr 2026 14:18:08 -0400 Subject: [PATCH 139/165] emphermal passover --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 1c8da78..5349e58 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -151,7 +151,7 @@ instance_groups: instances: 3 vm_extensions: - logs-opensearch-profile - - 15GB_ephemeral_disk + - 20GB_ephemeral_disk jobs: - name: bpm release: bpm From 31e79c7e60059efc0637a4c6e34539b645b7a1d4 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 30 Apr 2026 14:50:12 -0400 Subject: [PATCH 140/165] first run is only manager and dashboard --- opensearch-scaling-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-production.yml b/opensearch-scaling-production.yml index 9dcd8dd..56c7711 100644 --- a/opensearch-scaling-production.yml +++ b/opensearch-scaling-production.yml @@ -90,7 +90,7 @@ - type: replace path: /instance_groups/name=maintenance?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=maintenance?/vm_type? From 3fa348f38c92b8b5ca69508d6ef0b5c75252082e Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 08:07:46 -0400 Subject: [PATCH 141/165] platform s3 dejestor. this is supposed to pull from logsearch and put in opensearch --- opensearch-base.yml | 61 ++++++++++++++++++++++++++++++ opensearch-scaling-development.yml | 8 ++++ 2 files changed, 69 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 5349e58..b843243 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -396,6 +396,67 @@ instance_groups: update: serial: true # Block on this job to create deploy group 5 +- name: ingestor_s3 + instances: 0 + jobs: + - name: bpm + release: bpm + - name: opensearch + release: opensearch + consumes: *consumes-opensearch-manager + properties: + opensearch: + deployment_type: + - platform + heap_size: 1G + http_host: 127.0.0.1 + jvm_options: + - -Dlog4j2.formatMsgNoLookups=true + - consumes: *consumes-opensearch-manager + name: ingestor_s3 + properties: + logstash: + type: platform_recovery + jvm_options: + - -Dlog4j2.formatMsgNoLookups=true + queue: + max_bytes: 30gb + logstash_ingestor: + s3: + bucket: ((recovery_bucket)) + region: ((region)) + syslog_tls: + port: 6972 + ssl_cert: ((ingestor_syslog_server_tls.certificate)) + ssl_key: ((ingestor_syslog_server_tls.private_key)) + logstash_parser: + opensearch: + data_hosts: + - localhost + index: "logs-platform-%{+YYYY.MM.dd}" + index_type: '%{@type}' + ssl: + ca: ((opensearch_node.ca)) + certificate: ((logstash.certificate)) + private_key: ((logstash.private_key)) + provides: + ingestor: + as: ingestor_s3 + release: opensearch + - name: deployment_lookup_config + release: opensearch + azs: [z1,z2] + networks: + - name: services + persistent_disk_type: logs_opensearch_ingestor + stemcell: default + vm_extensions: + - logs-opensearch-ingestor-profile + - 20GB_ephemeral_disk + vm_type: t3.large + update: + serial: true # Block on this job to create deploy group 6 + ######################################################### # 9th deploy group - dashboards # Dashboard is ran last as it checks the types of other instances when restarting. all dashboard nodes must either be updated at the same diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 3082435..78683ab 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -63,8 +63,16 @@ path: /instance_groups/name=ingestor?/vm_type? value: m6i.large +- type: replace + path: /instance_groups/name=ingestor_s3?/instances? + value: 1 + +- type: replace + path: /instance_groups/name=ingestor_s3?/vm_type? + value: m6i.large + - type: replace path: /instance_groups/name=maintenance?/jobs?/name=smoke_tests?/properties?/smoke_tests?/count_test? value: From 86b8f3d881a404adc8ea16b4fda9dedb89efae0f Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 08:23:57 -0400 Subject: [PATCH 142/165] day work --- opensearch-scaling-development.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 78683ab..27af298 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -71,6 +71,9 @@ path: /instance_groups/name=ingestor_s3?/vm_type? value: m6i.large +- type: replace + path: /instance_groups/name=ingestor_s3?/prefix? + value: 2026/03/25/04/ - type: replace From 8d5c5cb1a4c47c67de437f52f4eaabdde84213fc Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 08:30:53 -0400 Subject: [PATCH 143/165] fixing ingestor need for notification --- opsfiles/add-keystore-credentials-errand.yml | 10 ++++++++++ opsfiles/opensearch-notification.yml | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/opsfiles/add-keystore-credentials-errand.yml b/opsfiles/add-keystore-credentials-errand.yml index b865f3f..566651e 100644 --- a/opsfiles/add-keystore-credentials-errand.yml +++ b/opsfiles/add-keystore-credentials-errand.yml @@ -58,3 +58,13 @@ path: /instance_groups/name=ingestor/jobs/name=opensearch/properties/opensearch?/notifications? value: <<: *add-credential-map + +- type: replace + path: /instance_groups/name=ingestor_s3/jobs/- + value: + <<: *add-keystore-credentials-errand + +- type: replace + path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch?/notifications? + value: + <<: *add-credential-map \ No newline at end of file diff --git a/opsfiles/opensearch-notification.yml b/opsfiles/opensearch-notification.yml index dd592f6..95cfe69 100644 --- a/opsfiles/opensearch-notification.yml +++ b/opsfiles/opensearch-notification.yml @@ -27,7 +27,6 @@ defaultkeystorepass: ((default_keystore_password)) - - type: replace path: /instance_groups/name=ingestor/jobs/name=opensearch/properties/opensearch/notification? value: From 7365edd22a55f212d9c08f9df87f60a3c5efd1d5 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 08:35:03 -0400 Subject: [PATCH 144/165] ingestor_s3 mssing --- opsfiles/opensearch-notification.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/opsfiles/opensearch-notification.yml b/opsfiles/opensearch-notification.yml index 95cfe69..03e9e63 100644 --- a/opsfiles/opensearch-notification.yml +++ b/opsfiles/opensearch-notification.yml @@ -38,3 +38,10 @@ value: keystorepass: ((keystore_password)) defaultkeystorepass: ((default_keystore_password)) + + +- type: replace + path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch/notification? + value: + keystorepass: ((keystore_password)) + defaultkeystorepass: ((default_keystore_password)) From a13c8e5e1e2f75e86ed6cc8cb332a8ac07f33d6a Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 08:45:16 -0400 Subject: [PATCH 145/165] ingestor_s3 mssing --- opsfiles/enable-node-tls.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/opsfiles/enable-node-tls.yml b/opsfiles/enable-node-tls.yml index 3e5a044..990df70 100644 --- a/opsfiles/enable-node-tls.yml +++ b/opsfiles/enable-node-tls.yml @@ -123,6 +123,23 @@ path: /instance_groups/name=ingestor/jobs/name=opensearch/properties?/opensearch?/http?/ssl? value: *http-tls-properties +# ingestor_s3 +- type: replace + path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties?/opensearch?/http_host? + value: 127.0.0.1 + +- type: replace + path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch?/admin? + value: *admin-tls-properties + +- type: replace + path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties/opensearch?/node?/ssl? + value: *node-tls-properties + +- type: replace + path: /instance_groups/name=ingestor_s3/jobs/name=opensearch/properties?/opensearch?/http?/ssl? + value: *http-tls-properties + # variables - type: replace path: /variables/name=opensearch_ca? From 01f820f6347cd54eaebf2473271488c121a0b8ba Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 09:03:18 -0400 Subject: [PATCH 146/165] giving correct permissions --- opensearch-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index b843243..582e4b5 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -451,7 +451,7 @@ instance_groups: persistent_disk_type: logs_opensearch_ingestor stemcell: default vm_extensions: - - logs-opensearch-ingestor-profile + - platform-opensearch-ingestor-profile - 20GB_ephemeral_disk vm_type: t3.large update: From bdcfabc212fa21653331991a2a7c0d1f3badf36d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 09:36:02 -0400 Subject: [PATCH 147/165] updating to make index direct for testing --- opensearch-base.yml | 3 ++- opensearch-scaling-development.yml | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 582e4b5..78499fa 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -425,6 +425,7 @@ instance_groups: s3: bucket: ((recovery_bucket)) region: ((region)) + prefix: 2026/03/25/04/ syslog_tls: port: 6972 ssl_cert: ((ingestor_syslog_server_tls.certificate)) @@ -433,7 +434,7 @@ instance_groups: opensearch: data_hosts: - localhost - index: "logs-platform-%{+YYYY.MM.dd}" + index: "jason" index_type: '%{@type}' ssl: ca: ((opensearch_node.ca)) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 27af298..78683ab 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -71,9 +71,6 @@ path: /instance_groups/name=ingestor_s3?/vm_type? value: m6i.large -- type: replace - path: /instance_groups/name=ingestor_s3?/prefix? - value: 2026/03/25/04/ - type: replace From 4a40e478962dd3b9b5e4374673b30c5ee04d1a5c Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 10:56:34 -0400 Subject: [PATCH 148/165] trying a different codec --- opensearch-base.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 78499fa..014654b 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -425,7 +425,12 @@ instance_groups: s3: bucket: ((recovery_bucket)) region: ((region)) - prefix: 2026/03/25/04/ + prefix: 2026/03/25/05/ + codec: + multiline: + pattern: ".*" + what: "previous" + negate: "false" syslog_tls: port: 6972 ssl_cert: ((ingestor_syslog_server_tls.certificate)) From 39e9517ae7a4be5e80fc946b286d26b847df8308 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 12:25:57 -0400 Subject: [PATCH 149/165] this fixes ingestion being one line --- opensearch-base.yml | 2 +- opensearch-scaling-development.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 014654b..b0ab987 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -363,7 +363,7 @@ instance_groups: temporary_directory: /var/vcap/data/ingestor_syslog/s3_temp # note that this is different than in the archivers # here, we're uploading the parsed message as json, not the raw log line - codec: json + codec: json_lines - plugin: opensearch options: {} deployment_dictionary: diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 78683ab..ebad293 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -65,7 +65,7 @@ - type: replace path: /instance_groups/name=ingestor_s3?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=ingestor_s3?/vm_type? From 3b1484b241a85e67431312409a849cf0049efeab Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 14:06:34 -0400 Subject: [PATCH 150/165] testing out branch --- opensearch-base.yml | 6 ------ opensearch-scaling-development.yml | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index b0ab987..7daa680 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -416,7 +416,6 @@ instance_groups: name: ingestor_s3 properties: logstash: - type: platform_recovery jvm_options: - -Dlog4j2.formatMsgNoLookups=true queue: @@ -426,11 +425,6 @@ instance_groups: bucket: ((recovery_bucket)) region: ((region)) prefix: 2026/03/25/05/ - codec: - multiline: - pattern: ".*" - what: "previous" - negate: "false" syslog_tls: port: 6972 ssl_cert: ((ingestor_syslog_server_tls.certificate)) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index ebad293..78683ab 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -65,7 +65,7 @@ - type: replace path: /instance_groups/name=ingestor_s3?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=ingestor_s3?/vm_type? From e0d11ba74bcd744a7b395f2241a4adea0f8e7c2e Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 14:55:08 -0400 Subject: [PATCH 151/165] giving a type --- opensearch-base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index 7daa680..a5f5dcc 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -416,6 +416,7 @@ instance_groups: name: ingestor_s3 properties: logstash: + type: new jvm_options: - -Dlog4j2.formatMsgNoLookups=true queue: From 04b1cabad8a15d2bc609450600848488da2e675f Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 May 2026 17:35:16 -0400 Subject: [PATCH 152/165] adding in remote hosts --- opensearch-base.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index a5f5dcc..e508b58 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -47,6 +47,9 @@ instance_groups: ip_addresses: true properties: opensearch: + reindex: + remote: + allowlist: ((elastic_master)) deployment_type: - platform node: @@ -99,6 +102,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + reindex: + remote: + allowlist: ((elastic_master)) deployment_type: - platform oidc: @@ -162,6 +168,9 @@ instance_groups: as: opensearch_manager properties: opensearch: + reindex: + remote: + allowlist: ((elastic_master)) deployment_type: - platform oidc: @@ -213,6 +222,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + reindex: + remote: + allowlist: ((elastic_master)) deployment_type: - platform oidc: @@ -319,6 +331,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + reindex: + remote: + allowlist: ((elastic_master)) deployment_type: - platform heap_size: 1G @@ -406,6 +421,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + reindex: + remote: + allowlist: ((elastic_master)) deployment_type: - platform heap_size: 1G @@ -474,6 +492,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: + reindex: + remote: + allowlist: ((elastic_master)) deployment_type: - platform oidc: From b29c8e86d598a93cd75930af077c8edc9229d9b9 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 May 2026 09:12:49 -0400 Subject: [PATCH 153/165] new manager --- opensearch-scaling-development.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 78683ab..39514e3 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 0 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? @@ -41,7 +41,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=opensearch_dashboards?/vm_type? @@ -57,7 +57,7 @@ - type: replace path: /instance_groups/name=ingestor?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=ingestor?/vm_type? @@ -65,7 +65,7 @@ - type: replace path: /instance_groups/name=ingestor_s3?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=ingestor_s3?/vm_type? @@ -104,7 +104,7 @@ - type: replace path: /instance_groups/name=maintenance?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=maintenance?/vm_type? From 1442cc11acdf3d8620579612626f9e801e6ac470 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 May 2026 10:10:05 -0400 Subject: [PATCH 154/165] adding in data and dashboard --- opensearch-scaling-development.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 39514e3..a590716 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? @@ -41,7 +41,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=opensearch_dashboards?/vm_type? From 68021b6af0dcda13628a24b6bf5134feb9ce1334 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 May 2026 14:21:24 -0400 Subject: [PATCH 155/165] adding region fix --- opensearch-base.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index e508b58..a07c92d 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -47,9 +47,9 @@ instance_groups: ip_addresses: true properties: opensearch: - reindex: - remote: - allowlist: ((elastic_master)) + repository: + s3: + region: ((region)) deployment_type: - platform node: @@ -102,9 +102,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - reindex: - remote: - allowlist: ((elastic_master)) + repository: + s3: + region: ((region)) deployment_type: - platform oidc: @@ -168,9 +168,9 @@ instance_groups: as: opensearch_manager properties: opensearch: - reindex: - remote: - allowlist: ((elastic_master)) + repository: + s3: + region: ((region)) deployment_type: - platform oidc: @@ -222,9 +222,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - reindex: - remote: - allowlist: ((elastic_master)) + repository: + s3: + region: ((region)) deployment_type: - platform oidc: @@ -331,9 +331,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - reindex: - remote: - allowlist: ((elastic_master)) + repository: + s3: + region: ((region)) deployment_type: - platform heap_size: 1G @@ -421,9 +421,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - reindex: - remote: - allowlist: ((elastic_master)) + repository: + s3: + region: ((region)) deployment_type: - platform heap_size: 1G @@ -492,9 +492,9 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - reindex: - remote: - allowlist: ((elastic_master)) + repository: + s3: + region: ((region)) deployment_type: - platform oidc: From a624272ea44b7a08ea229e011c9e5b388a52c128 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 08:48:24 -0400 Subject: [PATCH 156/165] adding in access --- opensearch-base.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/opensearch-base.yml b/opensearch-base.yml index a07c92d..1a4e6b2 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -50,6 +50,8 @@ instance_groups: repository: s3: region: ((region)) + access_key: ((repository_access_key)) + secrets_key: ((repository_secret_key)) deployment_type: - platform node: @@ -105,6 +107,8 @@ instance_groups: repository: s3: region: ((region)) + access_key: ((repository_access_key)) + secrets_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -171,6 +175,8 @@ instance_groups: repository: s3: region: ((region)) + access_key: ((repository_access_key)) + secrets_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -225,6 +231,8 @@ instance_groups: repository: s3: region: ((region)) + access_key: ((repository_access_key)) + secrets_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -334,6 +342,8 @@ instance_groups: repository: s3: region: ((region)) + access_key: ((repository_access_key)) + secrets_key: ((repository_secret_key)) deployment_type: - platform heap_size: 1G @@ -424,6 +434,8 @@ instance_groups: repository: s3: region: ((region)) + access_key: ((repository_access_key)) + secrets_key: ((repository_secret_key)) deployment_type: - platform heap_size: 1G @@ -495,6 +507,8 @@ instance_groups: repository: s3: region: ((region)) + access_key: ((repository_access_key)) + secrets_key: ((repository_secret_key)) deployment_type: - platform oidc: From 2ab39b06ca8c6183bf6b4a8660a74b60d9888217 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 09:00:14 -0400 Subject: [PATCH 157/165] its secret key not secrets --- opensearch-base.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 1a4e6b2..5291d41 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -51,7 +51,7 @@ instance_groups: s3: region: ((region)) access_key: ((repository_access_key)) - secrets_key: ((repository_secret_key)) + secret_key: ((repository_secret_key)) deployment_type: - platform node: @@ -108,7 +108,7 @@ instance_groups: s3: region: ((region)) access_key: ((repository_access_key)) - secrets_key: ((repository_secret_key)) + secret_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -176,7 +176,7 @@ instance_groups: s3: region: ((region)) access_key: ((repository_access_key)) - secrets_key: ((repository_secret_key)) + secret_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -232,7 +232,7 @@ instance_groups: s3: region: ((region)) access_key: ((repository_access_key)) - secrets_key: ((repository_secret_key)) + secret_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -343,7 +343,7 @@ instance_groups: s3: region: ((region)) access_key: ((repository_access_key)) - secrets_key: ((repository_secret_key)) + secret_key: ((repository_secret_key)) deployment_type: - platform heap_size: 1G @@ -435,7 +435,7 @@ instance_groups: s3: region: ((region)) access_key: ((repository_access_key)) - secrets_key: ((repository_secret_key)) + secret_key: ((repository_secret_key)) deployment_type: - platform heap_size: 1G @@ -508,7 +508,7 @@ instance_groups: s3: region: ((region)) access_key: ((repository_access_key)) - secrets_key: ((repository_secret_key)) + secret_key: ((repository_secret_key)) deployment_type: - platform oidc: From 085f7d25f5f9a6d06d404d9dc45910360c56a584 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 12:11:47 -0400 Subject: [PATCH 158/165] release candidate --- ci/pipeline.yml | 59 +++++++++++++++++++----------- opensearch-scaling-development.yml | 4 +- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 17a0aa9..85f1e3c 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -1,5 +1,16 @@ --- groups: + - name: all + jobs: + - build-opensearch-test-release + - deploy-opensearch-development + - upload-dashboards-objects-development + - smoke-tests-development + - deploy-opensearch-staging + - upload-dashboards-objects-staging + - smoke-tests-staging + - deploy-opensearch-production + - upload-dashboards-objects-production - name: development jobs: - build-opensearch-test-release @@ -14,6 +25,7 @@ groups: - name: production jobs: - deploy-opensearch-production + - upload-dashboards-objects-production jobs: - name: build-opensearch-test-release @@ -110,13 +122,6 @@ jobs: - deploy-logs-platform-config/opsfiles/add-keystore-credentials-errand.yml vars_files: - terraform-secrets/terraform.yml - # on_failure: - # put: slack - # params: - # <<: *slack-failure-params - # text: | - # :x: FAILED to deploy logs-platform in development - # <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> - name: upload-dashboards-objects-development serial_groups: [bosh-development] @@ -225,6 +230,7 @@ jobs: - get: opensearch-stemcell-jammy - get: deploy-logs-platform-config - get: opensearch-staging-deployment + passed: [deploy-opensearch-staging] trigger: true - task: upload-dashboards-objects file: pipeline-tasks/bosh-logs-errand.yml @@ -247,6 +253,7 @@ jobs: - get: opensearch-stemcell-jammy - get: deploy-logs-platform-config - get: opensearch-staging-deployment + passed: [deploy-opensearch-staging] trigger: true - task: smoke-tests file: pipeline-tasks/bosh-logs-errand.yml @@ -314,13 +321,35 @@ jobs: vars_files: - terraform-secrets/terraform.yml + - name: upload-dashboards-objects-production + serial_groups: [bosh-production] + plan: + - in_parallel: + - get: pipeline-tasks + - get: opensearch-test-release + - get: opensearch-stemcell-jammy + - get: deploy-logs-platform-config + - get: opensearch-production-deployment + passed: [deploy-opensearch-production] + trigger: true + - task: upload-dashboards-objects + file: pipeline-tasks/bosh-logs-errand.yml + params: + BOSH_ENVIRONMENT: ((bosh_production_environment)) + BOSH_CLIENT: ((bosh_client)) + BOSH_CLIENT_SECRET: ((bosh_production_client_secret)) + BOSH_DEPLOYMENT: logs-platform + BOSH_ERRAND: upload-dashboards-objects + BOSH_FLAGS: "--keep-alive" + BOSH_CA_CERT: ((bosh_ca_cert_store.certificate)) + resources: - name: opensearch-test-release-git-repo type: git source: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/opensearch-boshrelease - branch: platform + branch: main - name: opensearch-final-builds-dir-tarball type: s3-iam @@ -388,10 +417,7 @@ resources: commit_verification_keys: ((cloud-gov-pgp-keys)) uri: https://github.com/cloud-gov/cg-pipeline-tasks.git branch: main - # - name: slack - # type: slack-notification - # source: - # url: ((slack-webhook-url)) + - name: opensearch-development-deployment type: bosh-deployment source: &bosh-params-development @@ -438,15 +464,6 @@ resource_types: aws_region: us-gov-west-1 tag: latest - # - name: slack-notification - # type: registry-image - # source: - # aws_access_key_id: ((ecr_aws_key)) - # aws_secret_access_key: ((ecr_aws_secret)) - # repository: slack-notification-resource - # aws_region: us-gov-west-1 - # tag: latest - - name: bosh-deployment type: registry-image source: diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index a590716..39514e3 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 0 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? @@ -41,7 +41,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards?/instances? - value: 1 + value: 0 - type: replace path: /instance_groups/name=opensearch_dashboards?/vm_type? From 3f1e067f0b178beb3dbe6724cef86e118f788dee Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 12:31:06 -0400 Subject: [PATCH 159/165] adding back in data node --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 39514e3..da56b81 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From e1c1a1c2908ccba375de4ab999a5b42cbfc5e731 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 12:54:20 -0400 Subject: [PATCH 160/165] back to one --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index da56b81..39514e3 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 2 + value: 0 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From d0bf823433990f6fd87a4093e800596ffeb8ceca Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 13:42:07 -0400 Subject: [PATCH 161/165] removing snapshot --- opensearch-base.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index 5291d41..a5f5dcc 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -47,11 +47,6 @@ instance_groups: ip_addresses: true properties: opensearch: - repository: - s3: - region: ((region)) - access_key: ((repository_access_key)) - secret_key: ((repository_secret_key)) deployment_type: - platform node: @@ -104,11 +99,6 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - repository: - s3: - region: ((region)) - access_key: ((repository_access_key)) - secret_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -172,11 +162,6 @@ instance_groups: as: opensearch_manager properties: opensearch: - repository: - s3: - region: ((region)) - access_key: ((repository_access_key)) - secret_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -228,11 +213,6 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - repository: - s3: - region: ((region)) - access_key: ((repository_access_key)) - secret_key: ((repository_secret_key)) deployment_type: - platform oidc: @@ -339,11 +319,6 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - repository: - s3: - region: ((region)) - access_key: ((repository_access_key)) - secret_key: ((repository_secret_key)) deployment_type: - platform heap_size: 1G @@ -431,11 +406,6 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - repository: - s3: - region: ((region)) - access_key: ((repository_access_key)) - secret_key: ((repository_secret_key)) deployment_type: - platform heap_size: 1G @@ -504,11 +474,6 @@ instance_groups: consumes: *consumes-opensearch-manager properties: opensearch: - repository: - s3: - region: ((region)) - access_key: ((repository_access_key)) - secret_key: ((repository_secret_key)) deployment_type: - platform oidc: From b8dc1bf4cbe2efed87633a2813958bfb10ad8bd1 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 13:47:27 -0400 Subject: [PATCH 162/165] removing snapshot --- opensearch-scaling-development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index 39514e3..da56b81 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -9,7 +9,7 @@ - type: replace path: /instance_groups/name=opensearch_data?/instances? - value: 0 + value: 2 - type: replace path: /instance_groups/name=opensearch_data?/vm_type? From f0382fd9262f209d14a067ecc2b717173b358544 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 14:05:59 -0400 Subject: [PATCH 163/165] trying order of operation --- opensearch-scaling-development.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opensearch-scaling-development.yml b/opensearch-scaling-development.yml index da56b81..ebad293 100644 --- a/opensearch-scaling-development.yml +++ b/opensearch-scaling-development.yml @@ -41,7 +41,7 @@ - type: replace path: /instance_groups/name=opensearch_dashboards?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=opensearch_dashboards?/vm_type? @@ -57,7 +57,7 @@ - type: replace path: /instance_groups/name=ingestor?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=ingestor?/vm_type? @@ -104,7 +104,7 @@ - type: replace path: /instance_groups/name=maintenance?/instances? - value: 0 + value: 1 - type: replace path: /instance_groups/name=maintenance?/vm_type? From 20a1d7d565851d279302924a89f9057fc644952d Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 May 2026 16:31:53 -0400 Subject: [PATCH 164/165] updating timing --- opensearch-base.yml | 3 ++- opensearch-scaling-staging.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/opensearch-base.yml b/opensearch-base.yml index a5f5dcc..95d64fa 100644 --- a/opensearch-base.yml +++ b/opensearch-base.yml @@ -135,7 +135,7 @@ instance_groups: networks: - name: services update: - max_in_flight: 1 # Only update 1 data node at a time or risk downtime + max_in_flight: 2 serial: true # Block on this job to create deploy group 2 env: bosh: @@ -394,6 +394,7 @@ instance_groups: networks: - name: services update: + max_in_flight: 2 serial: true # Block on this job to create deploy group 5 - name: ingestor_s3 diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml index 06456b8..46ec45f 100644 --- a/opensearch-scaling-staging.yml +++ b/opensearch-scaling-staging.yml @@ -25,7 +25,7 @@ - type: replace path: /instance_groups/name=opensearch_old_data?/instances? - value: 4 + value: 0 - type: replace path: /instance_groups/name=opensearch_old_data?/vm_type? From c7b57ac00690d4825281a979257da6fc3ba29a4c Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 13 May 2026 09:12:44 -0400 Subject: [PATCH 165/165] should be respective environments --- opensearch-scaling-production.yml | 2 +- opensearch-scaling-staging.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensearch-scaling-production.yml b/opensearch-scaling-production.yml index 56c7711..dff6a55 100644 --- a/opensearch-scaling-production.yml +++ b/opensearch-scaling-production.yml @@ -110,7 +110,7 @@ aliases: - domain: nats.service.cf.internal targets: - - deployment: cf-development + - deployment: cf-production domain: bosh instance_group: nats network: default diff --git a/opensearch-scaling-staging.yml b/opensearch-scaling-staging.yml index 46ec45f..f76419c 100644 --- a/opensearch-scaling-staging.yml +++ b/opensearch-scaling-staging.yml @@ -110,7 +110,7 @@ aliases: - domain: nats.service.cf.internal targets: - - deployment: cf-development + - deployment: cf-staging domain: bosh instance_group: nats network: default