OmegaBI инструкция по установке на Astra Linux

#!/bin/bash
uname=$1
groupadd -r $uname
#adduser --system --home /data/$uname --shell /bin/false --no-create-home $uname
useradd -r -g $uname -d /data/$uname -s /usr/sbin/nologin $uname

#!/bin/bash
ACT=$1
BSS="
nginx
couchdb
postgresql
keycloak
druid
kafka
payara
nifi
"

if [[ -z $1 ]]; then ACT='status'; fi

for BS in $BSS; do
   echo $BS
   clear; systemctl $ACT $BS
   sleep 3
done

#/data/nifi/bin/nifi.sh $ACT

echo "Payara Control"
#su -c "/data/payara/bin/asadmin start-domain" payara
#su -c "/data/payara/bin/asadmin ping-connection-pool analytics" payara
#su -c "/data/payara/bin/asadmin list-applications" payara

JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre

apt install build-essential libpcre3-dev zlib1g-dev
cd /opt
wget https://www.openssl.org/source/old/1.1.1/openssl-1.1.1d.tar.gz
tar xvf openssl-1.1.1d.tar.gz
wget https://nginx.org/download/nginx-1.21.0.tar.gz
tar xvf nginx-1.21.0.tar.gz
cd nginx-1.21.0
./configure --with-openssl=/opt/openssl-1.1.1d --with-http_ssl_module --with-stream --with-http_v2_module --with-http_realip_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_auth_request_module --with-stream_ssl_module
make && make install

/usr/local/nginx/sbin/nginx -V
nginx version: nginx/1.21.0
built by gcc 6.3.0 20170516 (Debian 6.3.0-18+deb9u1)
configure arguments:

               "bio6-gkomega-ru": {
              "budgets": [
                {
                  "type": "anyComponentStyle",
                  "maximumWarning": "6kb"
                }
              ],
              "optimization": true,
              "outputHashing": "all",
              "sourceMap": false,
              "extractCss": true,
              "namedChunks": false,
              "aot": true,
              "extractLicenses": true,
              "vendorChunk": false,
              "buildOptimizer": true,
              "fileReplacements": [
                {
                  "replace": "src/environments/environment.ts",
                  "with": "src/environments/environment.bio6-gkomega-ru.ts"
                }
              ],
              "assets": [
                {
                  "glob": "favicon.ico",
                  "input": "src/assets/conf/bio6-gkomega-ru/",
                  "output": "/"
                },
                {
                  "input": "src/assets/",
                  "output": "/assets/",
                  "glob": "**/*",
                  "ignore": [
                    "conf/*.json",
                    "**/*.*~"
                  ]
                },
                {
                  "input": "src/assets/conf/bio6-gkomega-ru/",
                  "output": "/assets/",
                  "glob": "keycloak.json"
                }
              ]
            },

cat src/assets/conf/bio6-gkomega-ru/keycloak.json
{
  "realm": "analytics",
  "auth-server-url": "https://bio6.gkomega.ru/auth/",
  "ssl-required": "none",
  "resource": "app-analytics",
  "public-client": true,
  "verify-token-audience": true,
  "use-resource-role-mappings": true,
  "confidential-port": 0
}

   import { version } from '../../package.json';

   export const environment = {
     title: 'OmegaBI',
     production: true,
     version: version,
     host: 'https://bio6.gkomega.ru',
     base: 'https://bio6.gkomega.ru',
     api: 'https://bio6.gkomega.ru/arm/api',
     druid: {
       broker: 'https://bio6.gkomega.ru/druid/v2',
       coordinator: 'https://bio6.gkomega.ru/druid/coordinator/v1',
       indexer: 'https://bio6.gkomega.ru/druid/indexer/v1'
     },
     cdb: {
       api: "https://bio6.gkomega.ru/cdb",
       cred: "YWRtaW46YWRtaW4="
     },
     thetaSketchSize: 262144,
     restrictedColumns: [],
     defaults: {
       "pivot-report": {
         "статистика сайта ГИСП": {
           "columns": ["Год", "Месяц"],
           "rows": ["Тип ссылки"]
         }
       },
       "PLOTTING-TR-4": {
         "title": "Круговая диаграмма с детализацией",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Распорядитель",
         "drillLevel2": "Типология",
         "drillLevel3": "Учреждение",
         "allowCrossFilter": false,
       },
       "PLOTTING-BAR": {
         "title": "Столбчатая диаграмма с детализацией",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Распорядитель",
         "drillLevel2": "Типология",
         "drillLevel3": "Учреждение",
         "drillLevel4": ""
       },
       "plotting-hierarchy": {
         "title": "Иерархия",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Распорядитель", "Типология", "Учреждение"],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Распорядитель",
         "drillLevel2": "Типология",
         "drillLevel3": "Учреждение"
       },
       "plotting-heatmap": {
         "title": "Тепловая карта",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Учреждение", "Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "drillLevel1": "Группа должности",
         "drillLevel2": "",
         "drillLevel3": "",
       },
       "Voting": {
         "title": "Опросник",
         "dataSource": "Структура базового норматива затрат",
         "dimensions": ["НаименованиеГосУслуги"],
         "metrics": ["Доля", "Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort"
       },
       "Radar": {
         "title": "Радар",
         "dataSource": "Структура базового норматива затрат",
         "dimensions": ["Распорядитель", "ТипПоказателя"],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Do not sort",
         "groupBy": "ТипПоказателя",
         "sliderBy": "Распорядитель",
         "scale": "linear",
         "percentView": false,
         "showCenterSumm": false,
         "isPercentView": false,
         "allowCrossFilter": false,
         "showAxisLabels": true,
       },
       "PLOTTING-TR-8": {
         "title": "Датчик",
         "dataSource": "Принято Уволено от ДИТ",
         "dimensions": [],
         "reportType": "Over a period",
         "metrics": ["Уволено"]
       },
       "PLOTTING-TR-9": {
         "title": "Линейная диаграмма с детализацией",
         "dataSource": "Проект ПФХД",
         "dimensions": ["Этап "],
         "metrics": ["Сумма"],
         "reportType": "Over a period",
         "sortType": "Metric",
         "drillLevel1": "Учреждение",
         "drillLevel2": "НаименованиеВидаРасходов",
         "drillLevel3": "НаименованиеОСГУ",
         "drillLevel4": ""
       }
     },
     kc: {
       "realm": 'analytics',
       "clientId": 'app-analytics',
       "url": 'https://bio6.gkomega.ru/auth'
     }
   };

$ export NODE_OPTIONS="--max-old-space-size=4096"
ng build -c bio6-gkomega-ru

$ sudo apt-get install language-pack-ru
$ sudo locale-gen ru_RU.UTF-8
$ sudo localectl set-locale LANG=ru_RU.UTF-8
$ sudo apt install postgresql
$ sudo -u postgres
$ createdb analytics -E UTF8 -l ru_RU.UTF-8 -T template
$ sudo -u postgres psql
postgres=# CREATE USER analytics WITH PASSWORD '123456';
postgres=# GRANT ALL PRIVILEGES ON DATABASE analytics TO analytics;
postgres=# \q
$  psql -d analytics < analytics-210226.sql
$ echo -e "host\tanalytics\tanalytics\t10.33.8.52/24\tmd5" >> /etc/postgresql/9.6/main/pg_hba.conf
$ echo -e "listen_addresses = '*'" >> /etc/postgresql/9.6/main/postgresql.conf
$ exit
$ sudo systemctl restart postgresql

$ psql -d analytics < analytics.sql

analytics=# select * from olap_properties;
  id  |     code      |          name           |                    prop_value
------+---------------+-------------------------+--------------------------------------------------
 8551 | broker        | Брокер                  | http://omegabi.gkomega.ru/druid/v2
 8301 | indexer       | Индексатор              | http://omegabi.gkomega.ru/druid/indexer/v1
 8302 | coordinator   | Координатор             | http://omegabi.gkomega.ru/druid/coordinator/v1
 8552 | historical    | Исторический узел       | http://omegabi.gkomega.ru/druid/historical/v1
 8553 | middleManager | Менеджер среднего звена | http://omegabi.gkomega.ru/druid/middleManager/v1
(5 rows)

analytics=# update olap_properties set prop_value = 'https://omegabi2/druid/v2' where id = 8551;
analytics=# update olap_properties set prop_value = 'https://omegabi2/druid/indexer/v1' where id = 8301;
analytics=# update olap_properties set prop_value = 'https://omegabi2/druid/coordinator/v1' where id = 8302;
analytics=# update olap_properties set prop_value = 'https://omegabi2/druid/historical/v1' where id = 8552;
analytics=# update olap_properties set prop_value = 'https://omegabi2/druid/middleManager/v1' where id = 8553;

$ grep omegabi2 /etc/hosts
127.0.1.1       omegabi2

echo -e "host\tanalytics\tanalytics\t127.0.0.1/32\tmd5" >> /etc/postgresql/9.6/main/pg_hba.conf

psql -U analytics -h omegabi2 analytics
Пароль пользователя analytics:
psql: СБОЙ:  error obtaining MAC configuration for user "analytics"

# Return zero data instead of ENOENT/ENODATA in the absence of record
zero_if_notfound: yes

wget https://search.maven.org/remotecontent?filepath=fish/payara/distributions/payara/5.2020.2/payara-5.2020.2.zip -O payara-520.zip

#AS_JAVA="/usr/lib/jvm/java-11-openjdk-amd64"
# для Java 8 в Astra
AS_JAVA="/usr/lib/jvm/java-se-8u41-ri"

[Unit]
Description=Payara Server
After=network.target remote-fs.target

[Service]
User=payara
WorkingDirectory=/data/payara/glassfish
Environment=DOMAIN=domain1
#Environment=DOMAIN=production
#Environment=JAVA_PATH=/usr/lib/jvm/java-11-openjdk-amd64/bin
#Environment=PATH=/data/payara/glassfish/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-8-openjdk-amd64/bin:/usr/lib/jvm/java-8-openjdk-amd64/db/bin:/usr/lib/jvm/java-8-openjdk-amd64/jre/bin
Environment=PATH=/data/payara/glassfish/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-11-openjdk-amd64/bin:/usr/lib/jvm/java-11-openjdk-amd64/db/bin:/usr/lib/jvm/java-11-openjdk-amd64/jre/bin
#Environment=PATH=/data/payara/glassfish/bin:/data/payara/bin
Type=oneshot
RemainAfterExit=yes
ExecStart=/data/payara/glassfish/bin/asadmin start-domain $DOMAIN
ExecReload=/data/payara/glassfish/bin/asadmin restart-domain $DOMAIN
ExecStop=/data/payara/glassfish/bin/asadmin stop-domain $DOMAIN
TimeoutStartSec=300
TimeoutStopSec=30

[Install]
WantedBy=multi-user.target

cd /data/payara
wget https://jdbc.postgresql.org/download/postgresql-42.2.14.jar && mv postgresql-42.2.14.jar glassfish/domains/domain1/lib
chown -R payara:payara glassfish/domains/domain1/lib
systemctl restart payara

/data/payara/bin/asadmin create-jdbc-connection-pool --datasourceclassname org.postgresql.xa.PGXADataSource --restype javax.sql.XADataSource --property serverName=localhost:portNumber=5432:databaseName=analytics:user=analytics:password=123456 analytics

/data/payara/bin/asadmin  create-jdbc-resource --connectionpoolid analytics jdbc/analytics

#/data/payara/bin/asadmin deploy --force --name arm --contextroot arm OmegaBI-api-web-1.0-SNAPSHOT.war
/data/payara/bin/asadmin deploy --force --name analytics --contextroot analytics OmegaBI-api-web-1.0-SNAPSHOT.war

su - payara
cd ~/bin
./asadmin stop-domain
./asadmin start-domain
./asadmin ping-connection-pool analytics
./asadmin list-applications
./asadmin list-jvm-options

~/payara-5.201/bin/asadmin change-admin-password

~/payara-5.201/bin/asadmin --port 4848 enable-secure-admin
~/payara-5.201/bin/asadmin stop-domain
~/payara-5.201/bin/asadmin start-domain

/data/payara/bin/asadmin list-resource-refs
/data/payara/bin/asadmin delete-resource-ref jdbc/analytics
/data/payara/bin/asadmin list-jdbc-connection-pools
/data/payara/bin/asadmin  delete-jdbc-connection-pool --cascade=true analytics

cd /data/payara/work
git clone http://gitlab.gkomega.ru/omegabi/omegabi-api

cd omegabi-api/OmegaBI-api-web
mvn help:all-profiles

Non-resolvable parent POM for ru.spb.analytics:OmegaBI-api-web:1.0-SNAPSHOT: Could not find artifact ru.spb.analytics:OmegaBI-api:pom:1.0-SNAPSHOT and 'parent.relativePath' points at wrong local POM @ line 4, column 13

mvn clean install -P stage

/data/payara/glassfish/bin/asadmin deploy --force --name arm --contextroot arm target/OmegaBI-api-web-1.0-SNAPSHOT.war

/data/payara/glassfish/bin/asadmin undeploy arm

su -c "/data/payara/bin/asadmin list-applications" payara

sudo apt update && sudo apt install -y curl apt-transport-https gnupg dirmngr

wget https://downloads.apache.org/couchdb/KEYS
sudo gpg --import KEYS
mkdir /usr/share/keyrings/
curl https://couchdb.apache.org/repo/keys.asc | gpg --dearmor | sudo tee /usr/share/keyrings/couchdb-archive-keyring.gpg >/dev/null 2>&1
source /etc/os-release
echo "deb [trusted=yes] https://apache.jfrog.io/artifactory/couchdb-deb/  stretch main" > /etc/apt/sources.list.d/couchdb.list
sudo gpg --keyserver pgpkeys.mit.edu --recv-key  0EE62FB37A00258D

sudo apt update
sudo apt install -y couchdb jq

# curl -I http://localhost:5984
HTTP/1.1 200 OK
Cache-Control: must-revalidate
Content-Length: 247
Content-Type: application/json
Date: Fri, 04 Jun 2021 06:44:19 GMT
Server: CouchDB/3.1.1 (Erlang OTP/20)
X-Couch-Request-ID: 161930a7f3
X-CouchDB-Body-Time: 0

sudo apt install -y pkg-config erlang-reltool libicu-dev python3 python3-pip
sudo apt install -y libncurses5-dev libncursesw5-dev libmozjs-24-dev
sudo apt install -y python-sphinx 
pip3 install --upgrade sphinx_rtd_theme nose requests hypothesis

sudo apt-get install -y apt-transport-https gnupg ca-certificates git
cd /opt
git clone https://github.com/apache/couchdb.git
cd couchdb

$ curl -u admin:admin localhost:5984/_all_dbs | jq
  "_replicator",
  "_users",
  "grid",
  "panels",
  "ref-depts",
  "ref-grades",
  "ref-oiv-groups",
  "ref-oivs",
  "ref-ov-struc",
  "ref-position-families",
  "ref-positions"

## Вариант 2.
$ ls -1 /var/lib/couchdb/shards/00000000-7fffffff/*.couch | awk '{ FS="/"; $0=$0; print $7}'

$ for DBC in _replicator _users grid panels ref-depts ref-grades ref-oiv-groupd ref-oivs ref-ov-struc ref-position-families ref-positions; do curl -u admin:admin localhost:5984/$DBC/_all_docs?include_docs=true | jq '{"docs": [.rows[].doc]}' | jq 'del(.docs[]._rev)' > $DBC.json; done

$ for DBC in _replicator _users grid panels ref-depts ref-grades ref-oiv-groupd ref-oivs ref-ov-struc ref-position-families ref-positions; do curl -X PUT -u admin:admin localhost:5984/$DBC; done

for DBC in _replicator _users grid panels ref-depts ref-grades ref-oiv-groupd ref-oivs ref-ov-struc ref-position-families ref-positions; do curl -d @$DBC.json -H "Content-Type: application/json" -u admin:admin -X POST http://localhost:5984/$DBC/_bulk_docs; done

for DBC in panels; do curl -d @$DBC.json -H "Content-Type: application/json" -u admin:admin -X POST http://localhost:5984/$DBC/_bulk_docs; done

$ sed -e 's/127.0.0.1/0.0.0.0/' /opt/couchdb/etc/default.d/10-bind-address.ini
$ systemctl restart couchdb

curl -u admin:admin localhost:5984/_all_dbs | jq

cd /data; wget https://apache-mirror.rbc.ru/pub/apache/nifi/1.12.1/nifi-1.12.1-bin.tar.gz

user_add nifi
usermod -s /bin/bash nifi
chown -R nifi:nifi /data/nifi /var/log/nifi
su -c "/data/nifi/bin/nifi.sh start" nifi

su -c "/data/nifi/bin/nifi.sh status" nifi

Java home:
NiFi home: /opt/nifi

Bootstrap Config File: /opt/nifi/conf/bootstrap.conf

2021-02-02 10:06:43,383 INFO [main] org.apache.nifi.bootstrap.Command Apache NiFi is currently running, listening to Bootstrap on port 45349, PID=14601

[Unit]
Description=Apache Nifi Service

[Service]
Type=forking
User=nifi
Group=nifi
Environment=DRUID_SKIP_JAVA_CHECK=1
ExecStart=/data/nifi/bin/nifi.sh start
Restart=no

[Install]
WantedBy=multi-user.target

su -c "/data/nifi/bin/nifi.sh status" nifi
systemctl enable nifi
systemctl start nifi
systemctl status nifi

listeners=PLAINTEXT://localhost:9092

cd /data
wget https://github.com/keycloak/keycloak/releases/download/13.0.1/keycloak-13.0.1.tar.gz -O keycloak.tgz
#wget https://downloads.jboss.org/keycloak/13.0.1/keycloak-13.0.1.zip 
tar xvf keycloak.tgz
cd keycloak

user_add keycloak
mkdir -p /var/run/keycloak /etc/keycloak /var/log/keycloak
cp /data/keycloak/docs/contrib/scripts/systemd/launch.sh /data/keycloak/bin
cp /data/keycloak/docs/contrib/scripts/systemd/wildflyc.conf /etc/keycloak/
chown -R keycloak:keycloak /var/run/keycloak /etc/keycloak /var/log/keycloak 

[Unit]
Description=The Keycloak Server
After=syslog.target network.target
Before=httpd.service

[Service]
Environment=LAUNCH_JBOSS_IN_BACKGROUND=1
EnvironmentFile=/etc/keycloak/keycloak.conf
User=keycloak
Group=keycloak
LimitNOFILE=102642
PIDFile=/var/run/keycloak/keycloak.pid
ExecStart=/data/keycloak/bin/launch.sh $WILDFLY_MODE $WILDFLY_CONFIG $WILDFLY_BIND
StandardOutput=null

[Install]
WantedBy=multi-user.target

sed -i 's%WILDFLY_HOME="/opt/wildfly"%WILDFLY_HOME="/data/keycloak"%'  /data/keycloak/bin/launch.sh
systemctl enable keycloak
systemctl restart keycloak

/data/keycloak/bin/add-user-keycloak.sh -r master -u admin -p admin
/data/keycloak/bin/add-user.sh

# первый запуск
bin/standalone.sh

WARN [Coordinator-Exec--0] org.apache.druid.server.coordinator.rules.LoadRule - No available [_default_tier] servers or node capacity to assign segment

Command[zk] failed, see logfile for more details: /data/druid/var/sv/zk.log