Files
HaitiData/tasks.py
T
2020-07-02 11:29:32 +02:00

561 lines
21 KiB
Python

import ast
import json
import logging
import os
import re
import time
import datetime
import docker
import socket
from urlparse import urlparse
from invoke import run, task
BOOTSTRAP_IMAGE_CHEIP = 'codenvy/che-ip:nightly'
@task
def waitfordbs(ctx):
print "**************************databases*******************************"
ctx.run("/usr/bin/wait-for-databases {0}".format('db'), pty=True)
@task
def waitforgeoserver(ctx):
print "****************************geoserver********************************"
while not _rest_api_availability(os.environ['GEOSERVER_LOCATION'] + 'rest'):
print ("Wait for GeoServer API availability...")
print "GeoServer is available for HTTP calls!"
@task
def update(ctx):
print "***************************setting env*********************************"
ctx.run("env", pty=True)
pub_ip = _geonode_public_host_ip()
print "Public Hostname or IP is {0}".format(pub_ip)
pub_port = _geonode_public_port()
print "Public PORT is {0}".format(pub_port)
pub_protocol = 'https' if pub_port == '443' else 'http'
if pub_protocol == 'https' or pub_port == '80':
pub_port = None
db_url = _update_db_connstring()
geodb_url = _update_geodb_connstring()
service_ready = False
while not service_ready:
try:
socket.gethostbyname('geonode')
service_ready = True
except Exception:
time.sleep(10)
override_env = "$HOME/.override_env"
if os.path.exists(override_env):
os.remove(override_env)
else:
print("Can not delete the %s file as it doesn't exists" % override_env)
envs = {
"local_settings": "{0}".format(_localsettings()),
"siteurl": os.environ.get('SITEURL',
'{0}://{1}:{2}/'.format(pub_protocol, pub_ip, pub_port) if pub_port else '{0}://{1}/'.format(pub_protocol, pub_ip)),
"geonode_docker_host": "{0}".format(socket.gethostbyname('geonode')),
"public_protocol": pub_protocol,
"public_fqdn": "{0}{1}".format(pub_ip, ':' + pub_port if pub_port else ''),
"public_host": "{0}".format(pub_ip),
"dburl": os.environ.get('DATABASE_URL', db_url),
"geodburl": os.environ.get('GEODATABASE_URL', geodb_url),
"static_root": os.environ.get('STATIC_ROOT', '/mnt/volumes/statics/static/'),
"media_root": os.environ.get('MEDIA_ROOT', '/mnt/volumes/statics/uploaded/'),
"geoip_path": os.environ.get('GEOIP_PATH', '/mnt/volumes/statics/geoip.db'),
"monitoring": os.environ.get('MONITORING_ENABLED', True),
"monitoring_host_name": os.environ.get('MONITORING_HOST_NAME', 'geonode'),
"monitoring_service_name": os.environ.get('MONITORING_SERVICE_NAME', 'local-geonode'),
"monitoring_data_ttl": os.environ.get('MONITORING_DATA_TTL', 7),
"geonode_geodb_passwd": os.environ.get('GEONODE_GEODATABASE_PASSWORD', 'geonode_data'),
"default_backend_datastore": os.environ.get('DEFAULT_BACKEND_DATASTORE', 'datastore'),
"geonode_db_passwd": os.environ.get('GEONODE_DATABASE_PASSWORD', 'geonode'),
"geonode_geodb": os.environ.get('GEONODE_GEODATABASE', 'geonode_data'),
"db_url": os.environ.get('DATABASE_URL', 'postgres://geonode:geonode@db:5432/geonode'),
"geodb_url": os.environ.get('GEODATABASE_URL', 'postgis://geonode:geonode@db:5432/geonode_data'),
"geonode_db": os.environ.get('GEONODE_DATABASE', 'geonode'),
"gs_loc": os.environ.get('GEOSERVER_LOCATION', 'http://geoserver:8080/geoserver/'),
"gs_web_ui_loc": os.environ.get('GEOSERVER_WEB_UI_LOCATION',
'http://{0}:{1}/geoserver/'.format(pub_ip, pub_port) if pub_port else 'http://{0}/geoserver/'.format(pub_ip)),
"gs_pub_loc": os.environ.get('GEOSERVER_PUBLIC_LOCATION',
'http://{0}:{1}/geoserver/'.format(pub_ip, pub_port) if pub_port else 'http://{0}/geoserver/'.format(pub_ip)),
"gs_admin_pwd": os.environ.get('GEOSERVER_ADMIN_PASSWORD', 'geoserver'),
"override_fn": override_env
}
try:
current_allowed = ast.literal_eval(os.getenv('ALLOWED_HOSTS') or \
"['{public_fqdn}', '{public_host}', 'localhost', 'django', 'haitidata',]".format(**envs))
except ValueError:
current_allowed = []
current_allowed.extend(['{}'.format(pub_ip), '{}:{}'.format(pub_ip, pub_port)])
allowed_hosts = ['"{}"'.format(c) for c in current_allowed] + ['"geonode"', '"django"']
ctx.run("echo export DJANGO_SETTINGS_MODULE=\
{local_settings} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export MONITORING_ENABLED=\
{monitoring} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export MONITORING_HOST_NAME=\
{monitoring_host_name} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export MONITORING_SERVICE_NAME=\
{monitoring_service_name} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export MONITORING_DATA_TTL=\
{monitoring_data_ttl} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEONODE_GEODATABASE_PASSWORD=\
{geonode_geodb_passwd} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export DEFAULT_BACKEND_DATASTORE=\
{default_backend_datastore} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEONODE_DATABASE_PASSWORD=\
{geonode_db_passwd} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEONODE_GEODATABASE=\
{geonode_geodb} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export DATABASE_URL=\
{db_url} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEODATABASE_URL=\
{geodb_url} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEONODE_DATABASE=\
{geonode_db} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEOSERVER_LOCATION=\
{gs_loc} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEOSERVER_WEB_UI_LOCATION=\
{gs_web_ui_loc} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEOSERVER_PUBLIC_LOCATION=\
{gs_pub_loc} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEOSERVER_ADMIN_PASSWORD=\
{gs_admin_pwd} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export SITEURL=\
{siteurl} >> {override_fn}".format(**envs), pty=True)
ctx.run('echo export ALLOWED_HOSTS=\
"\\"{}\\"" >> {override_fn}'.format(allowed_hosts, **envs), pty=True)
ctx.run("echo export DATABASE_URL=\
{dburl} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEODATABASE_URL=\
{geodburl} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export STATIC_ROOT=\
{static_root} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export MEDIA_ROOT=\
{media_root} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export GEOIP_PATH=\
{geoip_path} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export LOGIN_URL=\
{siteurl}account/login/ >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export LOGOUT_URL=\
{siteurl}account/logout/ >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export LOGIN_REDIRECT_URL=\
{siteurl} >> {override_fn}".format(**envs), pty=True)
ctx.run("echo export LOGOUT_REDIRECT_URL=\
{siteurl} >> {override_fn}".format(**envs), pty=True)
ctx.run("source %s" % override_env, pty=True)
print "****************************finalize env**********************************"
ctx.run("env", pty=True)
@task
def migrations(ctx):
print "**************************migrations*******************************"
ctx.run("python manage.py makemigrations --noinput --merge --settings={0}".format(
_localsettings()
), pty=True)
ctx.run("python manage.py makemigrations --noinput --settings={0}".format(
_localsettings()
), pty=True)
ctx.run("python manage.py migrate --noinput --settings={0}".format(
_localsettings()
), pty=True)
ctx.run("python manage.py updategeoip --settings={0}".format(
_localsettings()
), pty=True)
try:
ctx.run("python manage.py rebuild_index --noinput --settings={0}".format(
_localsettings()
), pty=True)
except:
pass
@task
def statics(ctx):
print "**************************statics*******************************"
ctx.run('mkdir -p /mnt/volumes/statics/{static,uploads}')
ctx.run("python manage.py collectstatic --noinput --clear --settings={0}".format(
_localsettings()
), pty=True)
@task
def prepare(ctx):
print "**********************prepare fixture***************************"
ctx.run("rm -rf /tmp/default_oauth_apps_docker.json", pty=True)
_prepare_oauth_fixture()
ctx.run("rm -rf /tmp/default_site.json", pty=True)
_prepare_site_fixture()
# Updating OAuth2 Service Config
oauth_config = "/geoserver_data/data/security/filter/geonode-oauth2/config.xml"
ctx.run(
'sed -i "s|<cliendId>.*</cliendId>|<cliendId>{client_id}</cliendId>|g" {oauth_config}'.format(
client_id=os.environ['OAUTH2_CLIENT_ID'],
oauth_config=oauth_config
), pty=True)
ctx.run(
'sed -i "s|<clientSecret>.*</clientSecret>|<clientSecret>{client_secret}</clientSecret>|g" {oauth_config}'.format(
client_secret=os.environ['OAUTH2_CLIENT_SECRET'],
oauth_config=oauth_config
), pty=True)
@task
def fixtures(ctx):
print "**************************fixtures********************************"
ctx.run("python manage.py loaddata sample_admin \
--settings={0}".format(_localsettings()), pty=True)
ctx.run("python manage.py loaddata /tmp/default_oauth_apps_docker.json \
--settings={0}".format(_localsettings()), pty=True)
ctx.run("python manage.py loaddata /tmp/default_site.json \
--settings={0}".format(_localsettings()), pty=True)
ctx.run("python manage.py loaddata /usr/src/haitidata/fixtures/initial_data.json \
--settings={0}".format(_localsettings()), pty=True)
ctx.run("python manage.py set_all_layers_alternate \
--settings={0}".format(_localsettings()), pty=True)
ctx.run("python manage.py set_all_layers_metadata -d \
--settings={0}".format(_localsettings()), pty=True)
@task
def collectstatic(ctx):
print "************************static artifacts******************************"
ctx.run("django-admin.py collectstatic --noinput \
--settings={0}".format(_localsettings()), pty=True)
@task
def geoserverfixture(ctx):
print "********************geoserver fixture********************************"
_geoserver_info_provision(os.environ['GEOSERVER_LOCATION'] + "rest/")
@task
def monitoringfixture(ctx):
print "*******************monitoring fixture********************************"
ctx.run("rm -rf /tmp/default_monitoring_apps_docker.json", pty=True)
_prepare_monitoring_fixture()
try:
ctx.run("django-admin.py loaddata /tmp/default_monitoring_apps_docker.json \
--settings={0}".format(_localsettings()), pty=True)
except Exception as e:
print "ERROR installing monitoring fixture: " + str(e)
@task
def updategeoip(ctx):
print "**************************update geoip*******************************"
ctx.run("django-admin.py updategeoip \
--settings={0}".format(_localsettings()), pty=True)
@task
def updateadmin(ctx):
print "***********************update admin details**************************"
ctx.run("rm -rf /tmp/django_admin_docker.json", pty=True)
_prepare_admin_fixture(os.environ.get('ADMIN_PASSWORD', 'admin'), os.environ.get('ADMIN_EMAIL', 'admin@example.org'))
ctx.run("django-admin.py loaddata /tmp/django_admin_docker.json \
--settings={0}".format(_localsettings()), pty=True)
@task
def collectmetrics(ctx):
print "************************collect metrics******************************"
ctx.run("python -W ignore manage.py collect_metrics \
--settings={0} -n -t xml".format(_localsettings()), pty=True)
@task
def initialized(ctx):
print "**************************init file********************************"
ctx.run('date > /mnt/volumes/statics/geonode_init.lock')
def _docker_host_ip():
client = docker.from_env()
ip_list = client.containers.run(BOOTSTRAP_IMAGE_CHEIP,
network_mode='host'
).split("\n")
if len(ip_list) > 1:
print("Docker daemon is running on more than one \
address {0}".format(ip_list))
print("Only the first address:{0} will be returned!".format(
ip_list[0]
))
else:
print("Docker daemon is running at the following \
address {0}".format(ip_list[0]))
return ip_list[0]
def _container_exposed_port(component, instname):
client = docker.from_env()
ports_dict = json.dumps(
[c.attrs['Config']['ExposedPorts'] for c in client.containers.list(
filters={
'label': 'org.geonode.component={0}'.format(component),
'status': 'running'
}
) if '{0}'.format(instname) in c.name][0]
)
for key in json.loads(ports_dict):
port = re.split('/tcp', key)[0]
return port
def _update_db_connstring():
user = os.getenv('GEONODE_DATABASE', 'geonode')
pwd = os.getenv('GEONODE_DATABASE_PASSWORD', 'geonode')
dbname = os.getenv('GEONODE_DATABASE', 'geonode')
connstr = 'postgres://{0}:{1}@db:5432/{2}'.format(
user,
pwd,
dbname
)
return connstr
def _update_geodb_connstring():
geouser = os.getenv('GEONODE_GEODATABASE', 'geonode_data')
geopwd = os.getenv('GEONODE_GEODATABASE_PASSWORD', 'geonode_data')
geodbname = os.getenv('GEONODE_GEODATABASE', 'geonode_data')
geoconnstr = 'postgis://{0}:{1}@db:5432/{2}'.format(
geouser,
geopwd,
geodbname
)
return geoconnstr
def _localsettings():
settings = os.getenv('DJANGO_SETTINGS_MODULE', 'haitidata.settings')
return settings
def _rest_api_availability(url):
import requests
try:
r = requests.request('get', url, verify=False)
r.raise_for_status() # Raises a HTTPError if the status is 4xx, 5xxx
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
print "GeoServer connection error is {0}".format(e)
return False
except requests.exceptions.HTTPError as er:
print "GeoServer HTTP error is {0}".format(er)
return False
else:
print "GeoServer API are available!"
return True
def _geonode_public_host_ip():
gn_pub_hostip = os.getenv('GEONODE_LB_HOST_IP', None)
if not gn_pub_hostip:
gn_pub_hostip = _docker_host_ip()
return gn_pub_hostip
def _geonode_public_port():
gn_pub_port = os.getenv('GEONODE_LB_PORT', '')
if not gn_pub_port:
gn_pub_port = _container_exposed_port(
'nginx',
os.getenv('GEONODE_INSTANCE_NAME', 'geonode')
)
elif gn_pub_port in ('80', '443'):
gn_pub_port = None
return gn_pub_port
def _geoserver_info_provision(url):
from django.conf import settings
from geoserver.catalog import Catalog
print "Setting GeoServer Admin Password..."
cat = Catalog(url,
username=settings.OGC_SERVER_DEFAULT_USER,
password=settings.OGC_SERVER_DEFAULT_PASSWORD
)
headers = {
"Content-type": "application/xml",
"Accept": "application/xml"
}
data = """<?xml version="1.0" encoding="UTF-8"?>
<userPassword>
<newPassword>{0}</newPassword>
</userPassword>""".format(os.getenv('GEOSERVER_ADMIN_PASSWORD', 'geoserver'))
response = cat.http_request(cat.service_url + '/security/self/password', method="PUT", data=data, headers=headers)
print "Response Code: %s" % response.status_code
if response.status_code == 200:
print "GeoServer admin password updated SUCCESSFULLY!"
else:
print "WARNING: GeoServer admin password *NOT* updated: code [%s]" % response.status_code
def _prepare_oauth_fixture():
upurl = urlparse(os.environ['SITEURL'])
net_scheme = upurl.scheme
pub_ip = _geonode_public_host_ip()
print "Public Hostname or IP is {0}".format(pub_ip)
pub_port = _geonode_public_port()
print "Public PORT is {0}".format(pub_port)
default_fixture = [
{
"model": "oauth2_provider.application",
"pk": 1001,
"fields": {
"skip_authorization": True,
"created": "2018-05-31T10:00:31.661Z",
"updated": "2018-05-31T11:30:31.245Z",
"algorithm": "RS256",
"redirect_uris": "{0}://{1}:{2}/geoserver/index.html".format(net_scheme, pub_ip, pub_port) if pub_port else "{0}://{1}/geoserver/index.html".format(net_scheme, pub_ip),
"name": "GeoServer",
"authorization_grant_type": "authorization-code",
"client_type": "confidential",
"client_id": "{0}".format(os.environ['OAUTH2_CLIENT_ID']),
"client_secret": "{0}".format(os.environ['OAUTH2_CLIENT_SECRET']),
"user": [
"admin"
]
}
}
]
with open('/tmp/default_oauth_apps_docker.json', 'w') as fixturefile:
json.dump(default_fixture, fixturefile)
def _prepare_site_fixture():
upurl = urlparse(os.environ['SITEURL'])
default_fixture = [
{
"model": "sites.site",
"pk": 1,
"fields": {
"domain": "{0}".format(upurl.hostname),
"name": "{0}".format(upurl.hostname)
}
}
]
with open('/tmp/default_site.json', 'w') as fixturefile:
json.dump(default_fixture, fixturefile)
def _prepare_monitoring_fixture():
upurl = urlparse(os.environ['SITEURL'])
net_scheme = upurl.scheme
net_loc = upurl.netloc
pub_ip = _geonode_public_host_ip()
print "Public Hostname or IP is {0}".format(pub_ip)
pub_port = _geonode_public_port()
print "Public PORT is {0}".format(pub_port)
#d = str(datetime.datetime.now())
d = '1970-01-01 00:00:00'
default_fixture = [
{
"fields": {
"active": True,
"ip": "{0}".format(socket.gethostbyname('geonode')),
"name": "{0}".format(os.environ['MONITORING_HOST_NAME'])
},
"model": "monitoring.host",
"pk": 1
},
{
"fields": {
"active": True,
"ip": "{0}".format(socket.gethostbyname('geoserver')),
"name": "geoserver"
},
"model": "monitoring.host",
"pk": 2
},
{
"fields": {
"name": "{0}".format(os.environ['MONITORING_SERVICE_NAME']),
"url": "{0}://{1}/".format(net_scheme, net_loc),
"notes": "",
"last_check": d,
"active": True,
"host": 1,
"check_interval": "00:01:00",
"service_type": 1
},
"model": "monitoring.service",
"pk": 1
},
{
"fields": {
"name": "geoserver-hostgeonode",
"url": "{0}://{1}/".format(net_scheme, net_loc),
"notes": "",
"last_check": d,
"active": True,
"host": 1,
"check_interval": "00:01:00",
"service_type": 3
},
"model": "monitoring.service",
"pk": 2
},
{
"fields": {
"name": "geoserver-hostgeoserver",
"url": "{0}".format(os.environ['GEOSERVER_LOCATION']),
"notes": "",
"last_check": d,
"active": True,
"host": 2,
"check_interval": "00:01:00",
"service_type": 4
},
"model": "monitoring.service",
"pk": 3
},
{
"fields": {
"name": "default-geoserver",
"url": "{0}".format(os.environ['GEOSERVER_LOCATION']),
"notes": "",
"last_check": d,
"active": True,
"host": 2,
"check_interval": "00:01:00",
"service_type": 2
},
"model": "monitoring.service",
"pk": 4
}
]
with open('/tmp/default_monitoring_apps_docker.json', 'w') as fixturefile:
json.dump(default_fixture, fixturefile)
def _prepare_admin_fixture(admin_password, admin_email):
from django.contrib.auth.hashers import make_password
d = datetime.datetime.now()
mdext_date = d.isoformat()[:23] + "Z"
default_fixture = [
{
"fields": {
"date_joined": mdext_date,
"email": admin_email,
"first_name": "",
"groups": [],
"is_active": True,
"is_staff": True,
"is_superuser": True,
"last_login": mdext_date,
"last_name": "",
"password": make_password(admin_password),
"user_permissions": [],
"username": "admin"
},
"model": "people.Profile",
"pk": 1000
}
]
with open('/tmp/django_admin_docker.json', 'w') as fixturefile:
json.dump(default_fixture, fixturefile)