Merge pull request #110 from dynjnelson/master

revamp cluster initialization, fix usage of user/owner/runas/db_user, add dependencies

_modules/postgres_ext.py

@@ -174,7 +174,7 @@ def tablespace_exists(name, user=None, host=None, port=None, maintenance_db=None
     return name in tablespaces
 
 
-def tablespace_create(name, location, user=None, host=None, port=None,
+def tablespace_create(name, location, owner=None, user=None, host=None, port=None,
               maintenance_db=None, password=None, runas=None):
     '''
     Adds a tablespace to the Postgres server.
@@ -185,7 +185,9 @@ def tablespace_create(name, location, user=None, host=None, port=None,
 
         salt '*' postgres_ext.tablespace_create tablespacename '/path/datadir'
     '''
-    query = 'CREATE TABLESPACE {0} LOCATION \'{1}\''.format(name, location)
+    query = 'CREATE TABLESPACE "{0}" LOCATION \'{1}\''.format(name, location)
+    if owner is not None:
+        query += ' OWNER "{1}"'.format(owner)
 
     # Execute the command
     ret = _psql_prepare_and_run(['-c', query],
@@ -212,16 +214,16 @@ def tablespace_alter(name, user=None, host=None, port=None, maintenance_db=None,
     queries = []
 
     if new_name:
-        queries.append('ALTER TABLESPACE {} RENAME TO {}'.format(
+        queries.append('ALTER TABLESPACE "{}" RENAME TO "{}"'.format(
                        name, new_name))
     if new_owner:
-        queries.append('ALTER TABLESPACE {} OWNER TO {}'.format(
+        queries.append('ALTER TABLESPACE "{}" OWNER TO "{}"'.format(
                        name, new_owner))
     if set_option:
-        queries.append('ALTER TABLESPACE {} SET ({} = {})'.format(
+        queries.append('ALTER TABLESPACE "{}" SET ({} = {})'.format(
                        name, set_option[0], set_option[1]))
     if reset_option:
-        queries.append('ALTER TABLESPACE {} RESET ({})'.format(
+        queries.append('ALTER TABLESPACE "{}" RESET ({})'.format(
                        name, reset_option))
 
     for query in queries:
@@ -243,7 +245,7 @@ def tablespace_remove(name, user=None, host=None, port=None,
     .. code-block:: bash
         salt '*' postgres_ext.tablespace_remove tsname
     '''
-    query = 'DROP TABLESPACE {}'.format(name)
+    query = 'DROP TABLESPACE "{}"'.format(name)
     ret = _psql_prepare_and_run(['-c', query],
                                 user=user,
                                 host=host,
@@ -252,3 +254,4 @@ def tablespace_remove(name, user=None, host=None, port=None,
                                 maintenance_db=maintenance_db,
                                 password=password)
     return ret['retcode'] == 0
+

pillar.example

@@ -1,17 +1,19 @@
 postgres:
   pg_hba.conf: salt://postgres/pg_hba.conf
-  commands:
+  prepare_cluster:
-    initdb: service postgresql initdb
+    user: root
+    command: service postgresql initdb
+    test: test -f /path/to/some/file
+    env:
+      LC_ALL: en_US.UTF-8
 
   use_upstream_repo: False
 
-  lookup:
+  pkg: 'postgresql-9.3'
-    pkg: 'postgresql-9.3'
+  pkg_client: 'postgresql-client-9.3'
-    pkg_client: 'postgresql-client-9.3'
+  pkgs_extra:
-    pkgs_extra:
+    - postgresql-contrib
-      - postgresql-contrib
+    - postgresql-plpython
-      - postgresql-plpython
-    pg_hba: '/etc/postgresql/9.3/main/pg_hba.conf'
 
   users:
     localUser:
@@ -55,8 +57,8 @@ postgres:
       owner: 'localUser'
       user: 'localUser'
       template: 'template0'
-      lc_ctype: 'C.UTF-8'
+      lc_ctype: 'en_US.UTF-8'
-      lc_collate: 'C.UTF-8'
+      lc_collate: 'en_US.UTF-8'
       # optional schemas to enable on database
       schemas:
         uuid_ossp:
@@ -69,8 +71,8 @@ postgres:
       owner: 'localUser'
       user: 'remoteUser'
       template: 'template0'
-      lc_ctype: 'C.UTF-8'
+      lc_ctype: 'en_US.UTF-8'
-      lc_collate: 'C.UTF-8'
+      lc_collate: 'en_US.UTF-8'
       tablespace: 'my_space'
       # optional extensions to enable on database
       # extensions:

postgres/codenamemap.yaml

@@ -1,9 +1,12 @@
 {%- macro ubuntu_block(name, version) %}
 {{ name }}:
-  version: {{ version }}
   pkg_repo: deb http://apt.postgresql.org/pub/repos/apt/ {{ name }}-pgdg main
   conf_dir: /etc/postgresql/{{ version }}/main
-  data_dir: /var/lib/postgresql/{{ version }}/main
+  prepare_cluster:
+    command: pg_createcluster {{ version }} main
+    test: test -f /var/lib/postgresql/{{ version }}/main/PG_VERSION && test -f /etc/postgresql/{{ version }}/main/postgresql.conf
+    user: root
+    env: {}
 {%- endmacro %}
 
 {{ ubuntu_block('wheezy', 9.1) }}

postgres/defaults.yaml

@@ -7,8 +7,6 @@ postgres:
   python: python-psycopg2
   service: postgresql
   conf_dir: /var/lib/pgsql/data
-  data_dir: /var/lib/pgsql/data
-  version: 9.1
   use_upstream_repo: False
   users: {}
   acls: []
@@ -17,10 +15,16 @@ postgres:
   postgresconf_backup: True
   postgresconf: ""
   pg_hba.conf: salt://postgres/pg_hba.conf
-  initdb: True
-  initdb_user: root
-  initdb_args: --data-checksum
-  commands:
-    initdb: service postgresql initdb
   user: postgres
   group: postgres
+  # if prepare_cluster is over-ridden in any of:
+  # - osmap.yaml
+  # - oscodenamemap.yaml
+  # - osfingermap.yaml
+  # you will have to specify a complete dictionary.
+  prepare_cluster:
+    user: root
+    command: service postgresql initdb
+    test: test -f /var/lib/pgsql/data/PG_VERSION
+    env: {}
+

postgres/init.sls

@@ -16,9 +16,7 @@ postgresql-config-dir:
     - makedirs: True
     - require:
       - pkg: postgresql-installed
-{% if postgres.conf_dir == postgres.data_dir %}
       - cmd: postgresql-cluster-prepared
-{% endif %}
 
 postgresql-installed:
   pkg.installed:
@@ -28,20 +26,17 @@ postgresql-installed:
 # make sure the data directory and contents have been initialized
 postgresql-cluster-prepared:
   cmd.run:
-    {% if postgres.initdb %}
-    - name: {{ postgres.commands.initdb }} {{ postgres.initdb_args }} -D {{ postgres.data_dir }}
-    {% elif grains.os_family == 'Debian' %}
-    - name: pg_createcluster {{ postgres.version }} main
-    {# else: TODO #}
-    {% endif %}
     - cwd: /
-    - user: {{ postgres.initdb_user }}
+    - name: {{ postgres.prepare_cluster.command }}
+    - user: {{ postgres.prepare_cluster.user }}
     - unless:
-      - test -f {{ postgres.data_dir }}/PG_VERSION
+      - {{ postgres.prepare_cluster.test }}
     - require:
       - pkg: postgresql-installed
     - env:
-      LC_ALL: C.UTF-8
+{% for name, value in postgres.prepare_cluster.env.items() %}
+        {{ name }}: {{ value }}
+{% endfor %}
 
 postgresql-running:
   service.running:
@@ -96,6 +91,9 @@ postgresql-user-{{ name }}:
   postgres_user.absent:
     - name: {{ name }}
     - user: {{ user.get('runas', postgres.user) }}
+{% if user.get('user') %}
+    - db_user: {{ user.user }}
+{% endif %}
 {% else %}
   postgres_user.present:
     - name: {{ name }}
@@ -104,20 +102,26 @@ postgresql-user-{{ name }}:
     - createuser: {{ user.get('createuser', False) }}
     - inherit: {{ user.get('inherit', True) }}
     - replication: {{ user.get('replication', False) }}
-    - password: {{ user.get('password', 'changethis') }}
+    - password: {{ user.password }}
-    - user: {{ user.get('runas', postgres.user) }}
     - superuser: {{ user.get('superuser', False) }}
+    - user: {{ user.get('runas', postgres.user) }}
+{% if user.get('user') %}
+    - db_user: {{ user.get('runas', postgres.user) }}
+{% endif %}
 {% endif %}
     - require:
       - service: postgresql-running
+{% if user.get('user') %}
+      - postgres_user: postgresql-user-{{ user.user }}
+{% endif %}
 {% endfor %}
 
-{% for name, directory in postgres.tablespaces.items()  %}
+{% for name, tblspace in postgres.tablespaces.items()  %}
-postgresql-tablespace-dir-perms-{{ directory}}:
+postgresql-tablespace-dir-perms-{{ tblspace.directory}}:
   file.directory:
-    - name: {{ directory }}
+    - name: {{ tblspace.directory }}
-    - user: postgres
+    - user: {{ postgres.user }}
-    - group: postgres
+    - group: {{ postgres.group }}
     - makedirs: True
     - recurse:
       - user
@@ -126,11 +130,17 @@ postgresql-tablespace-dir-perms-{{ directory}}:
 postgresql-tablespace-{{ name }}:
   postgres_tablespace.present:
     - name: {{ name }}
-    - directory: {{ directory }}
+    - directory: {{ tblspace.directory }}
-    - user: postgres
+    - user: {{ tblspace.get('runas', postgres.user) }}
+{% if tblspace.get('user') %}
+    - db_user: {{ tblspace.user }}
+{% endif %}
+{% if tblspace.get('owner') %}
+    - owner: {{ tblspace.owner }}
+{% endif %}
     - require:
       - service: postgresql-running
-      - file: postgresql-tablespace-dir-perms-{{ directory}}
+      - file: postgresql-tablespace-dir-perms-{{ tblspace.directory}}
 {% endfor %}
 
 {% for name, db in postgres.databases.items()  %}
@@ -138,58 +148,115 @@ postgresql-db-{{ name }}:
 {% if db.get('ensure', 'present') == 'absent' %}
   postgres_database.absent:
     - name: {{ name }}
+    - user: {{ db.get('runas', postgres.user) }}
+{% if db.get('user') %}
+    - db_user: {{ db.user }}
+{% endif %}
     - require:
       - service: postgresql-running
 {% else %}
   postgres_database.present:
     - name: {{ name }}
-    - encoding: {{ db.get('encoding', 'UTF8') }}
+    {% if 'encoding' in db %}
-    - lc_ctype: {{ db.get('lc_ctype', 'en_US.UTF8') }}
+    - encoding: {{ db.encoding }}
-    - lc_collate: {{ db.get('lc_collate', 'en_US.UTF8') }}
+    {% endif %}
+    {% if 'lc_ctype' in db %}
+    - lc_ctype: {{ db.lc_type }}
+    {% endif %}
+    {% if 'lc_collate' in db %}
+    - lc_collate: {{ db.lc_collate }}
+    {% endif %}
     - template: {{ db.get('template', 'template0') }}
     - tablespace: {{ db.get('tablespace', 'pg_default') }}
     {% if db.get('owner') %}
-    - owner: {{ db.get('owner') }}
+    - owner: {{ db.owner }}
     {% endif %}
     - user: {{ db.get('runas', postgres.user) }}
+    {% if db.get('user') %}
+    - db_user: {{ db.user }}
+    {% endif %}
     - require:
       - service: postgresql-running
     {% if db.get('user') %}
-      - postgres_user: postgresql-user-{{ db.get('user') }}
+      - postgres_user: postgresql-user-{{ db.user }}
+    {% endif %}
+    {% if db.get('owner') %}
+      - postgres_user: postgresql-user-{{ db.owner }}
+    {% endif %}
+    {% if db.get('tablespace') %}
+      - postgres_tablespace: postgresql-tablespace-{{ name }}
     {% endif %}
 
-{% if db.schemas is defined %}
+{# NOTE: postgres_schema doesn't have a 'runas' equiv. at all #}
-{% for schema, schema_args in db.schemas.items() %}
+{% for schema_name, schema in db.get('schemas', dict()).items() %}
-postgresql-schema-{{ schema }}-for-db-{{ name }}:
+postgresql-schema-{{ schema_name }}-for-db-{{ name }}:
+{% if schema.get('ensure', 'present') == 'absent' %}
+  postgres_schema.absent:
+    - name: {{ schema_name }}
+    {% if schema.get('user') %}
+    - db_user: {{ schema.user }}
+    {% endif %}
+    - require:
+      - service: postgresql-running
+{% else %}
   postgres_schema.present:
-    - name: {{ schema }}
+    - name: {{ schema_name }}
     - dbname: {{ name }}
-{% if schema_args is not none %}
+    {% if schema.get('user') %}
-{% for arg, value in schema_args.items() %}
+    - db_user: {{ schema.user }}
-    - {{ arg }}: {{ value }}
+    {% endif %}
-{% endfor %}
+{% if schema.get('owner') %}
+    - owner: {{ schema.owner }}
+{% endif %}
     - require:
       - service: postgresql-running
+      - postgres_database: postgresql-db-{{ name }}
+    {% if schema.get('user') %}
+      - postgres_user: postgresql-user-{{ schema.user }}
+    {% endif %}
+    {% if schema.get('owner') %}
+      - postgres_user: postgresql-user-{{ schema.owner }}
+    {% endif %}
 {% endif %}
 {% endfor %}
-{% endif %}
 
-{% if db.extensions is defined %}
+{% for ext_name, ext in db.get('extensions', dict()).items() %}
-{% for ext, ext_args in db.extensions.items() %}
+postgresql-ext-{{ ext_name }}-for-db-{{ name }}:
-postgresql-ext-{{ ext }}-for-db-{{ name }}:
+  {% if ext.get('ensure', 'present') == 'absent' %}
+  postgres_extension.absent:
+    - name: {{ ext_name }}
+    - user: {{ ext.get('runas', postgres.user) }}
+    {% if ext.get('user') %}
+    - db_user: {{ ext.user }}
+    {% endif %}
+    - require:
+      - service: postgresql-running
+  {% else %}
   postgres_extension.present:
-    - name: {{ ext }}
+    - name: {{ ext_name }}
-    - user: {{ db.get('runas', postgres.user) }}
+    - user: {{ ext.get('runas', postgres.user) }}
+    {% if ext.get('user') %}
+    - db_user: {{ ext.user }}
+    {% endif %}
+    {% if ext.get('version') %}
+    - ext_version: {{ ext.version }}
+    {% endif %}
+    {% if ext.get('schema') %}
+    - schema: {{ ext.schema }}
+    {% endif %}
     - maintenance_db: {{ name }}
-{% if ext_args is not none %}
-{% for arg, value in ext_args.items() %}
-    - {{ arg }}: {{ value }}
-{% endfor %}
     - require:
       - service: postgresql-running
-{% endif %}
+      - postgres_database: postgresql-db-{{ name }}
+    {% if ext.get('user') %}
+      - postgres_user: postgresql-user-{{ ext.user }}
+    {% endif %}
+    {% if ext.get('schema') %}
+      - postgres_schema: postgresql-schema-{{ ext.schema }}
+    {% endif %}
+  {% endif %}
 {% endfor %}
-{% endif %}
+
 {% endif %}
 {% endfor %}
 

postgres/map.jinja

@@ -1,22 +1,22 @@
 {% import_yaml "postgres/defaults.yaml" as defaults %}
 {% import_yaml "postgres/osmap.yaml" as osmap %}
-{% import_yaml "postgres/codenamemap.yaml" as codemap %}
+{% import_yaml "postgres/codenamemap.yaml" as oscodenamemap %}
+{% import_yaml "postgres/osfingermap.yaml" as osfingermap %}
 
-{# get the settings for the os_family grain #}
+{% set _postgres = salt['grains.filter_by'](
-{% set osfam = salt['grains.filter_by'](osmap, grain='os_family') or {} %}
+    osmap,
-{# get the settings for the oscodename grain, os_family data will override
+    grain='os_family',
-    oscodename data #}
+    merge=salt['grains.filter_by'](
-{% set oscode = salt['grains.filter_by'](codemap,
+        oscodenamemap,
-                                         grain='oscodename',
+        grain='oscodename',
-                                         merge=osfam) or {} %}
+        merge=salt['grains.filter_by'](
+            osfingermap,
+            grain='osfinger',
+            merge=salt['pillar.get']('postgres', {})
+        )
+    )
+) %}
 
-{# merge the os family/codename specific data over the defaults #}
+{% do defaults.postgres.update(_postgres) %}
-{% do defaults.postgres.update(oscode) %}
 
-{# merge the pillar:lookup dict into the defaults/os specific dict #}
+{% set postgres = defaults.postgres %}
-{% set lookup = salt['pillar.get']('postgres:lookup',
-                                   default=defaults.postgres,
-                                   merge=True) %}
-
-{# merge the actual postgres pillar into the above combined dict #}
-{% set postgres = salt['pillar.get']('postgres', default=lookup, merge=True) %}

postgres/osfingermap.yaml

@@ -0,0 +1 @@
+{}

postgres/osmap.yaml

@@ -1,29 +1,34 @@
 RedHat:
-  initdb_user: postgres
-  commands:
-    initdb: initdb
   pkg: postgresql-server
   pkg_client: postgresql
   pkg_repo: pgdg94
   repo_baseurl: http://yum.postgresql.org/9.4/redhat/rhel-$releasever-$basearch
+  prepare_cluster:
+    test: test -f /var/lib/pgsql/data/PG_VERSION
+    env: {}
+{% if grains['os_family'] == 'RedHat' %}
+{% if grains['osmajorrelease'] >= 7 %}
+    user: root
+    command: postgresql-setup initdb
+{% else %}
+    user: postgres
+    command: initdb -D /var/lib/pgsql/data
+{% endif %}
+{% endif %}
 Arch:
   conf_dir: /var/lib/postgres/data
-  data_dir: /var/lib/postgres/data
+  prepare_cluster:
-  initdb_user: postgres
+    user: postgres
-  initdb_args: --locale en_US.UTF8 -E UTF8
+    command: initdb -D /var/lib/postgresql/data
-  commands:
+    test: test -f /var/lib/postgres/data/PG_VERSION
-    initdb: initdb
+    env: {}
   pkg_client: postgresql
   pkg_dev: postgresql
 Debian:
   pkg_repo_file: /etc/apt/sources.list.d/pgdg.list
   pkg_dev: postgresql-server-dev-all
   pkg_libpq_dev: libpq-dev
-  initdb: False
 Suse:
-  initdb_user: postgres
-  commands:
-    initdb: initdb
   pkg: postgresql-server
   pkg_client: postgresql
 FreeBSD: