OmegaBI analytics deploy

front (10.0.4.138): openresty, keycloak, druid, payara, kafka
back (10.0.4.212): postgresql, couchdb, nifi

server {
    server_name bio6.gkomega.ru;

    location / {
        proxy_pass  http://10.33.8.28;
        include proxy_params;
   }

    listen 443 ssl;

    include /etc/nginx/ssl/gkomega_wc.conf;
    include /etc/letsencrypt/options-ssl-nginx.conf;
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;

}

server {
    if ($host = bio6.gkomega.ru) {
        return 301 https://$host$request_uri;
    }

    listen 80;
    server_name bio6.gkomega.ru;
    return 404;

}

JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre

cd /data/openresty/work/
git clone http://gitlab.gkomega.ru/omegabi/omegabi-v02.git

npm install -g @angular/cli
cd /data/openresty/work/omegabi-v02
npm install

               "bio6-gkomega-ru": {
              "budgets": [
                {
                  "type": "anyComponentStyle",
                  "maximumWarning": "6kb"
                }
              ],
              "optimization": true,
              "outputHashing": "all",
              "sourceMap": false,
              "extractCss": true,
              "namedChunks": false,
              "aot": true,
              "extractLicenses": true,
              "vendorChunk": false,
              "buildOptimizer": true,
              "fileReplacements": [
                {
                  "replace": "src/environments/environment.ts",
                  "with": "src/environments/environment.bio6-gkomega-ru.ts"
                }
              ],
              "assets": [
                {
                  "glob": "favicon.ico",
                  "input": "src/assets/conf/bio6-gkomega-ru/",
                  "output": "/"
                },
                {
                  "input": "src/assets/",
                  "output": "/assets/",
                  "glob": "**/*",
                  "ignore": [
                    "conf/*.json",
                    "**/*.*~"
                  ]
                },
                {
                  "input": "src/assets/conf/bio6-gkomega-ru/",
                  "output": "/assets/",
                  "glob": "keycloak.json"
                }
              ]
            },

cat src/assets/conf/bio6-gkomega-ru/keycloak.json
{
  "realm": "analytics",
  "auth-server-url": "https://bio6.gkomega.ru/auth/",
  "ssl-required": "none",
  "resource": "app-analytics",
  "public-client": true,
  "verify-token-audience": true,
  "use-resource-role-mappings": true,
  "confidential-port": 0
}

   import { version } from '../../package.json';

   export const environment = {
     title: 'OmegaBI',
     production: true,
     version: version,
     host: 'https://bio6.gkomega.ru',
     base: 'https://bio6.gkomega.ru',
     api: 'https://bio6.gkomega.ru/arm/api',
     druid: {
       broker: 'https://bio6.gkomega.ru/druid/v2',
       coordinator: 'https://bio6.gkomega.ru/druid/coordinator/v1',
       indexer: 'https://bio6.gkomega.ru/druid/indexer/v1'
     },
     cdb: {
       api: "https://bio6.gkomega.ru/cdb",
       cred: "YWRtaW46YWRtaW4="
     },
     thetaSketchSize: 262144,
     restrictedColumns: [],
     defaults: {
       "pivot-report": {
         "статистика сайта ГИСП": {
           "columns": ["Год", "Месяц"],
           "rows": ["Тип ссылки"]
         }
       },
       "PLOTTING-TR-4": {
         "title": "Круговая диаграмма с детализацией",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Распорядитель",
         "drillLevel2": "Типология",
         "drillLevel3": "Учреждение",
         "allowCrossFilter": false,
       },
       "PLOTTING-BAR": {
         "title": "Столбчатая диаграмма с детализацией",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Распорядитель",
         "drillLevel2": "Типология",
         "drillLevel3": "Учреждение",
         "drillLevel4": ""
       },
       "plotting-hierarchy": {
         "title": "Иерархия",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Распорядитель", "Типология", "Учреждение"],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Распорядитель",
         "drillLevel2": "Типология",
         "drillLevel3": "Учреждение"
       },
       "plotting-heatmap": {
         "title": "Тепловая карта",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Учреждение", "Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Группа должности",
         "drillLevel2": "",
         "drillLevel3": "",
       },
       "Voting": {
         "title": "Опросник",
         "dataSource": "Структура базового норматива затрат",
         "dimensions": ["НаименованиеГосУслуги"],
         "metrics": ["Доля", "Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort"
       },
       "Radar": {
         "title": "Радар",
         "dataSource": "Структура базового норматива затрат",
         "dimensions": ["Распорядитель", "ТипПоказателя"],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "groupBy": "ТипПоказателя",
         "sliderBy": "Распорядитель",
         "scale": "linear",
         "percentView": false,
         "showCenterSumm": false,
         "isPercentView": false,
         "allowCrossFilter": false,
         "showAxisLabels": true,
       },
       "PLOTTING-TR-8": {
         "title": "Датчик",
         "dataSource": "Принято Уволено от ДИТ",
         "dimensions": [],
         "reportType": "Over a period",
         "metrics": ["Уволено"]
       },
       "PLOTTING-TR-9": {
         "title": "Линейная диаграмма с детализацией",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Metric",
         "drillLevel1": "Учреждение",
         "drillLevel2": "НаименованиеВидаРасходов",
         "drillLevel3": "НаименованиеОСГУ",
         "drillLevel4": ""
       }
     },
     kc: {
       "realm": 'analytics',
       "clientId": 'app-analytics',
       "url": 'https://bio6.gkomega.ru/auth'
     }
   };

   $ export NODE_OPTIONS="--max-old-space-size=4096"
   ng build -c bio6-gkomega-ru

ng update @angular/core@11.2.14
или
ng update --force @angular/core@11.2.14

$ sudo apt-get install language-pack-ru
$ sudo locale-gen ru_RU.UTF-8
$ sudo localectl set-locale LANG=ru_RU.UTF-8
$ sudo apt install postgresql
$ sudo -u postgres
$ createdb analytics -E UTF8 -l en_US.UTF-8 -T template
$ sudo -u postgres psql
postgres=# CREATE USER analytics WITH PASSWORD '123456';
postgres=# GRANT ALL PRIVILEGES ON DATABASE analytics TO analytics;
postgres=# \q
$  psql -d analytics < analytics-210226.sql
$ echo -e "host\tanalytics\tanalytics\t10.33.8.52/24\tmd5" >> /etc/postgresql/9.5/main/pg_hba.conf
$ echo -e "listen_addresses = '*'" >> /etc/postgresql/9.5/main/postgresql.conf
$ exit
$ sudo systemctl restart postgresql

$ psql -d analytics < analytics.sql

analytics=# select * from olap_properties;
  id  |     code      |          name           |                    prop_value
------+---------------+-------------------------+--------------------------------------------------
 8551 | broker        | Брокер                  | http://omegabi.gkomega.ru/druid/v2
 8301 | indexer       | Индексатор              | http://omegabi.gkomega.ru/druid/indexer/v1
 8302 | coordinator   | Координатор             | http://omegabi.gkomega.ru/druid/coordinator/v1
 8552 | historical    | Исторический узел       | http://omegabi.gkomega.ru/druid/historical/v1
 8553 | middleManager | Менеджер среднего звена | http://omegabi.gkomega.ru/druid/middleManager/v1
(5 rows)

update olap_properties set prop_value = 'http://bio6/druid/v2' where id = 8551;
update olap_properties set prop_value = 'http://bio6/druid/indexer/v1' where id = 8301;
update olap_properties set prop_value = 'http://bio6/druid/coordinator/v1' where id = 8302;
update olap_properties set prop_value = 'http://bio6/druid/historical/v1' where id = 8552;
update olap_properties set prop_value = 'http://bio6/druid/middleManager/v1' where id = 8553;

### 
update olap_properties set prop_value = 'https://newomegabi.gkomega.ru/druid/v2' where id = 8551;
update olap_properties set prop_value = 'https://newomegabi.gkomega.ru/druid/indexer/v1' where id = 8301;
update olap_properties set prop_value = 'https://newomegabi.gkomega.ru/druid/coordinator/v1' where id = 8302;
update olap_properties set prop_value = 'https://newomegabi.gkomega.ru/druid/historical/v1' where id = 8552;
update olap_properties set prop_value = 'https://newomegabi.gkomega.ru/druid/middleManager/v1' where id = 8553;

$ grep bio6 /etc/hosts
127.0.1.1       bio6

wget https://search.maven.org/remotecontent?filepath=fish/payara/distributions/payara/5.2020.2/payara-5.2020.2.zip -O payara-520.zip

AS_JAVA="/usr/lib/jvm/java-11-openjdk-amd64"

[Unit]
Description=Payara Server
After=network.target remote-fs.target

[Service]
User=payara
WorkingDirectory=/data/payara/glassfish
Environment=DOMAIN=domain1
#Environment=DOMAIN=production
#Environment=JAVA_PATH=/usr/lib/jvm/java-11-openjdk-amd64/bin
#Environment=PATH=/data/payara/glassfish/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-8-openjdk-amd64/bin:/usr/lib/jvm/java-8-openjdk-amd64/db/bin:/usr/lib/jvm/java-8-openjdk-amd64/jre/bin
Environment=PATH=/data/payara/glassfish/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-11-openjdk-amd64/bin:/usr/lib/jvm/java-11-openjdk-amd64/db/bin:/usr/lib/jvm/java-11-openjdk-amd64/jre/bin
#Environment=PATH=/data/payara/glassfish/bin:/data/payara/bin
Type=oneshot
RemainAfterExit=yes
ExecStart=/data/payara/glassfish/bin/asadmin start-domain $DOMAIN
ExecReload=/data/payara/glassfish/bin/asadmin restart-domain $DOMAIN
ExecStop=/data/payara/glassfish/bin/asadmin stop-domain $DOMAIN
TimeoutStartSec=300
TimeoutStopSec=30

[Install]
WantedBy=multi-user.target

$ wget https://jdbc.postgresql.org/download/postgresql-42.2.14.jar && cp postgresql-42.2.14.jar payara-5.2020.2/glassfish/domain/domain1/lib

asadmin> create-jdbc-connection-pool –datasourceclassname org.postgresql.xa.PGXADataSource –restype javax.sql.XADataSource –property serverName=host:portNumber=5432:databaseName=analytics:user=analytics:password=**** analytics

asadmin> create-jdbc-resource --connectionpoolid analytics jdbc/analytics

$ ./bin/asadmin deploy --force --name arm –contextroot arm analytics-spb-api-web-1.0-SNAPSHOT.war

su - payara
cd ~/bin
./asadmin stop-domain
./asadmin start-domain
./asadmin ping-connection-pool analytics
./asadmin list-applications
./asadmin list-jvm-options

~/payara-5.201/bin/asadmin change-admin-password

~/payara-5.201/bin/asadmin --port 4848 enable-secure-admin
~/payara-5.201/bin/asadmin stop-domain
~/payara-5.201/bin/asadmin start-domain

cd /data/payara/work
git clone http://gitlab.gkomega.ru/omegabi/omegabi-api

cd omegabi-api/OmegaBI-api-web
mvn help:all-profiles

Non-resolvable parent POM for ru.spb.analytics:OmegaBI-api-web:1.0-SNAPSHOT: Could not find artifact ru.spb.analytics:OmegaBI-api:pom:1.0-SNAPSHOT and 'parent.relativePath' points at wrong local POM @ line 4, column 13

mvn clean install -P stage

/data/payara/glassfish/bin/asadmin deploy --force --name arm --contextroot arm target/OmegaBI-api-web-1.0-SNAPSHOT.war

/data/payara/glassfish/bin/asadmin undeploy arm

su -c "/data/payara/bin/asadmin list-applications" payara

bin/asdamin create-jdbc-connection-pool --datasourceclassname org.postgresql.xa.PGXADataSource --restype javax.sql.XADataSource --property serverName=localhost:portNumber=5432:databaseName=analytics:user=analytics:password=123456 analytics

sudo apt-get install -y apt-transport-https gnupg ca-certificates
echo "deb https://apache.bintray.com/couchdb-deb xenial main"     | sudo tee -a /etc/apt/sources.list.d/couchdb.list
sudo apt update
sudo apt install -y couchdb

$ curl -u admin:admin localhost:5984/_all_dbs | jq
  "_replicator",
  "_users",
  "grid",
  "panels",
  "ref-depts",
  "ref-grades",
  "ref-oiv-groups",
  "ref-oivs",
  "ref-ov-struc",
  "ref-position-families",
  "ref-positions"

## Вариант 2.
$ ls -1 /var/lib/couchdb/shards/00000000-7fffffff/*.couch | awk '{ FS="/"; $0=$0; print $7}'

$ for DBC in _replicator _users grid panels ref-depts ref-grades ref-oiv-groupd ref-oivs ref-ov-struc ref-position-families ref-positions; do curl -u admin:admin localhost:5984/$DBC/_all_docs?include_docs=true | jq '{"docs": [.rows[].doc]}' | jq 'del(.docs[]._rev)' > $DBC.json; done

$ for DBC in _replicator _users grid panels ref-depts ref-grades ref-oiv-groupd ref-oivs ref-ov-struc ref-position-families ref-positions; do curl -X PUT -u admin:admin localhost:5984/$DBC; done

for DBC in _replicator _users grid panels ref-depts ref-grades ref-oiv-groupd ref-oivs ref-ov-struc ref-position-families ref-positions; do curl -d @$DBC.json -H "Content-Type: application/json" -u admin:admin -X POST http://localhost:5984/$DBC/_bulk_docs; done

for DBC in panels; do curl -d @$DBC.json -H "Content-Type: application/json" -u admin:admin -X POST http://localhost:5984/$DBC/_bulk_docs; done

$ sed -e 's/127.0.0.1/0.0.0.0/' /opt/couchdb/etc/default.d/10-bind-address.ini
$ systemctl restart couchdb

cd /opt; wget https://apache-mirror.rbc.ru/pub/apache/nifi/1.12.1/nifi-1.12.1-bin.tar.gz

/opt/nifi/bin/nifi.sh start

root@bi-setup-test:/opt# /opt/nifi/bin/nifi.sh status
nifi.sh: JAVA_HOME not set; results may vary

Java home:
NiFi home: /opt/nifi

Bootstrap Config File: /opt/nifi/conf/bootstrap.conf

2021-02-02 10:06:43,383 INFO [main] org.apache.nifi.bootstrap.Command Apache NiFi is currently running, listening to Bootstrap on port 45349, PID=14601

listeners=PLAINTEXT://localhost:9092

$ /home/keycloak/keycloak-11.0.0/bin# ./add-user-keycloak.sh -r master -u admin -p 123456
$ systemctl restart keycloak
$ /home/keycloak/keycloak-11.0.0/bin/add-user.sh

WARN [Coordinator-Exec--0] org.apache.druid.server.coordinator.rules.LoadRule - No available [_default_tier] servers or node capacity to assign segment

Command[zk] failed, see logfile for more details: /data/druid/var/sv/zk.log