parent
cb8c309a9c
commit
ef6fc4213c
@ -1 +1,50 @@ |
|||||||
iface: eth1 |
iface: eth1 |
||||||
|
|
||||||
|
ha_enabled: False |
||||||
|
|
||||||
|
hadoop: |
||||||
|
|
||||||
|
#Variables for <core-site_xml> - common |
||||||
|
|
||||||
|
fs_default_FS_port: 8020 |
||||||
|
nameservice_id: mycluster3 |
||||||
|
|
||||||
|
#Variables for <hdfs-site_xml> |
||||||
|
|
||||||
|
dfs_permissions_superusergroup: hdfs |
||||||
|
dfs_namenode_name_dir: |
||||||
|
- /namedir1/ |
||||||
|
- /namedir2/ |
||||||
|
dfs_replication: 3 |
||||||
|
dfs_namenode_handler_count: 50 |
||||||
|
dfs_blocksize: 67108864 |
||||||
|
dfs_datanode_data_dir: |
||||||
|
- /datadir1/ |
||||||
|
- /datadir2/ |
||||||
|
dfs_datanode_address_port: 50010 |
||||||
|
dfs_datanode_http_address_port: 50075 |
||||||
|
dfs_datanode_ipc_address_port: 50020 |
||||||
|
dfs_namenode_http_address_port: 50070 |
||||||
|
dfs_ha_zkfc_port: 8019 |
||||||
|
qjournal_port: 8485 |
||||||
|
qjournal_http_port: 8480 |
||||||
|
dfs_journalnode_edits_dir: /journaldir/ |
||||||
|
zookeeper_clientport: 2181 |
||||||
|
zookeeper_leader_port: 2888 |
||||||
|
zookeeper_election_port: 3888 |
||||||
|
|
||||||
|
#Variables for <mapred-site_xml> - common |
||||||
|
mapred_job_tracker_ha_servicename: myjt3 |
||||||
|
mapred_job_tracker_http_address_port: 50030 |
||||||
|
mapred_task_tracker_http_address_port: 50060 |
||||||
|
mapred_job_tracker_port: 8021 |
||||||
|
mapred_ha_jobtracker_rpc-address_port: 8023 |
||||||
|
mapred_ha_zkfc_port: 8018 |
||||||
|
mapred_job_tracker_persist_jobstatus_dir: /jobdir/ |
||||||
|
mapred_local_dir: |
||||||
|
- /mapred1/ |
||||||
|
- /mapred2/ |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,47 +0,0 @@ |
|||||||
|
|
||||||
hadoop: |
|
||||||
|
|
||||||
#Variables for <core-site.xml> - common |
|
||||||
|
|
||||||
fs.default.FS.port: 8020 |
|
||||||
nameservice.id: mycluster2 |
|
||||||
|
|
||||||
#Variables for <hdfs-site.xml> |
|
||||||
|
|
||||||
dfs.permissions.superusergroup: hdfs |
|
||||||
dfs_namenode_name_dir: |
|
||||||
- /namedir1/ |
|
||||||
- /namedir2/ |
|
||||||
dfs.replication: 3 |
|
||||||
dfs.namenode.handler.count: 50 |
|
||||||
dfs.blocksize: 67108864 |
|
||||||
dfs_datanode_data_dir: |
|
||||||
- /datadir1/ |
|
||||||
- /datadir2/ |
|
||||||
dfs.datanode.address.port: 50010 |
|
||||||
dfs.datanode.http.address.port: 50075 |
|
||||||
dfs.datanode.ipc.address.port: 50020 |
|
||||||
dfs.namenode.http.address.port: 50070 |
|
||||||
dfs.ha.zkfc.port: 8019 |
|
||||||
qjournal.port: 8485 |
|
||||||
qjournal.http.port: 8480 |
|
||||||
dfs_journalnode_edits_dir: /journaldir/ |
|
||||||
zookeeper.clientport: 2181 |
|
||||||
zookeeper.leader_port: 2888 |
|
||||||
zookeeper.election_port: 3888 |
|
||||||
|
|
||||||
#Variables for <mapred-site.xml> - common |
|
||||||
mapred.job.tracker.ha.servicename: myjt2 |
|
||||||
mapred.job.tracker.http.address.port: 50030 |
|
||||||
mapred.task.tracker.http.address.port: 50060 |
|
||||||
mapred.job.tracker.port: 8021 |
|
||||||
mapred.ha.jobtracker.rpc-address.port: 8023 |
|
||||||
mapred.ha.zkfc.port: 8018 |
|
||||||
mapred_job_tracker_persist_jobstatus_dir: /jobdir/ |
|
||||||
mapred_local_dir: |
|
||||||
- /mapred1/ |
|
||||||
- /mapred2/ |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,5 +1,5 @@ |
|||||||
--- |
--- |
||||||
# The playbook for common tasks |
# The playbook for common tasks |
||||||
|
|
||||||
- include: common.yml tags=slaves,no_ha |
- include: common.yml tags=slaves |
||||||
|
|
||||||
|
@ -0,0 +1,38 @@ |
|||||||
|
--- |
||||||
|
# Playbook for Hadoop master servers |
||||||
|
|
||||||
|
- name: Install the namenode and jobtracker packages |
||||||
|
yum: name={{ item }} state=installed |
||||||
|
with_items: |
||||||
|
- hadoop-0.20-mapreduce-jobtracker |
||||||
|
- hadoop-hdfs-namenode |
||||||
|
|
||||||
|
- name: Copy the hadoop configuration files for no ha |
||||||
|
template: src=roles/common/templates/hadoop_conf/{{ item }}.j2 dest=/etc/hadoop/conf/{{ item }} |
||||||
|
with_items: |
||||||
|
- core-site.xml |
||||||
|
- hadoop-metrics.properties |
||||||
|
- hadoop-metrics2.properties |
||||||
|
- hdfs-site.xml |
||||||
|
- log4j.properties |
||||||
|
- mapred-site.xml |
||||||
|
- slaves |
||||||
|
- ssl-client.xml.example |
||||||
|
- ssl-server.xml.example |
||||||
|
notify: restart hadoop master services |
||||||
|
|
||||||
|
- name: Create the data directory for the namenode metadata |
||||||
|
file: path={{ item }} owner=hdfs group=hdfs state=directory |
||||||
|
with_items: hadoop.dfs_namenode_name_dir |
||||||
|
|
||||||
|
- name: Format the namenode |
||||||
|
shell: creates=/usr/lib/hadoop/namenode.formatted su - hdfs -c "hadoop namenode -format"; touch /usr/lib/hadoop/namenode.formatted |
||||||
|
|
||||||
|
- name: start hadoop namenode services |
||||||
|
service: name=hadoop-hdfs-namenode state=started |
||||||
|
|
||||||
|
- name: Give permissions for mapred users |
||||||
|
shell: creates=/usr/lib/hadoop/fs.initialized su - hdfs -c "hadoop fs -chown hdfs:hadoop /"; su - hdfs -c "hadoop fs -chmod 0774 /"; touch /usr/lib/hadoop/namenode.initialized |
||||||
|
|
||||||
|
- name: start hadoop jobtracker services |
||||||
|
service: name=hadoop-0.20-mapreduce-jobtracker state=started |
@ -1,5 +1,9 @@ |
|||||||
--- |
--- |
||||||
# Playbook for Hadoop master primary servers |
# Playbook for Hadoop master primary servers |
||||||
|
|
||||||
- include: hadoop_master.yml tags=no_ha |
- include: hadoop_master.yml |
||||||
|
when: ha_enabled |
||||||
|
|
||||||
|
- include: hadoop_master_no_ha.yml |
||||||
|
when: not ha_enabled |
||||||
|
|
||||||
|
@ -1,73 +0,0 @@ |
|||||||
--- |
|
||||||
# Playbook for Hadoop master secondary server |
|
||||||
|
|
||||||
|
|
||||||
- name: Install the namenode and jobtracker packages |
|
||||||
yum: name=${item} state=installed |
|
||||||
with_items: |
|
||||||
- hadoop-0.20-mapreduce-jobtrackerha |
|
||||||
- hadoop-hdfs-namenode |
|
||||||
- hadoop-hdfs-zkfc |
|
||||||
- hadoop-0.20-mapreduce-zkfc |
|
||||||
|
|
||||||
- name: Copy the hadoop configuration files |
|
||||||
template: src=roles/common/templates/hadoop_ha_conf/${item}.j2 dest=/etc/hadoop/conf/${item} |
|
||||||
with_items: |
|
||||||
- core-site.xml |
|
||||||
- hadoop-metrics.properties |
|
||||||
- hadoop-metrics2.properties |
|
||||||
- hdfs-site.xml |
|
||||||
- log4j.properties |
|
||||||
- mapred-site.xml |
|
||||||
- slaves |
|
||||||
- ssl-client.xml.example |
|
||||||
- ssl-server.xml.example |
|
||||||
when_unset: $ha_disabled |
|
||||||
notify: restart hadoopha master services |
|
||||||
|
|
||||||
- name: Create the data directory for the namenode metadata |
|
||||||
file: path=${item} owner=hdfs group=hdfs state=directory |
|
||||||
with_items: ${hadoop.dfs_namenode_name_dir} |
|
||||||
|
|
||||||
- name: Create the data directory for the jobtracker ha |
|
||||||
file: path=${item} owner=mapred group=mapred state=directory |
|
||||||
with_items: ${hadoop.mapred_job_tracker_persist_jobstatus_dir} |
|
||||||
|
|
||||||
|
|
||||||
- name: Initialize the secodary namenode |
|
||||||
shell: creates=/usr/lib/hadoop/namenode.formatted su - hdfs -c "hadoop namenode -bootstrapStandby"; touch /usr/lib/hadoop/namenode.formatted |
|
||||||
|
|
||||||
- name: start hadoop namenode services |
|
||||||
service: name=${item} state=started |
|
||||||
with_items: |
|
||||||
- hadoop-hdfs-namenode |
|
||||||
|
|
||||||
- name: Initialize the zkfc for namenode |
|
||||||
shell: creates=/usr/lib/hadoop/zkfc.formatted su - hdfs -c "hdfs zkfc -formatZK"; touch /usr/lib/hadoop/zkfc.formatted |
|
||||||
register: nn_result |
|
||||||
|
|
||||||
- name: restart zkfc for namenode |
|
||||||
service: name=hadoop-hdfs-zkfc state=restarted |
|
||||||
delegate_to: ${item} |
|
||||||
with_items: ${groups.hadoop_masters} |
|
||||||
when_set: $nn_result and $nn_result.changed |
|
||||||
|
|
||||||
- name: Give permissions for mapred users |
|
||||||
shell: creates=/usr/lib/hadoop/fs.initialized su - hdfs -c "hadoop fs -chown hdfs:hadoop /"; su - hdfs -c "hadoop fs -chmod 0774 /"; touch /usr/lib/hadoop/namenode.initialized |
|
||||||
|
|
||||||
- name: Initialize the zkfc for jobtracker |
|
||||||
shell: creates=/usr/lib/hadoop/zkfcjob.formatted su - mapred -c "hadoop mrzkfc -formatZK"; touch /usr/lib/hadoop/zkfcjob.formatted |
|
||||||
register: jt_result |
|
||||||
|
|
||||||
- name: restart zkfc for jobtracker |
|
||||||
service: name=hadoop-0.20-mapreduce-zkfc state=restarted |
|
||||||
delegate_to: ${item} |
|
||||||
with_items: ${groups.hadoop_masters} |
|
||||||
when_set: $jt_result and $jt_result.changed |
|
||||||
|
|
||||||
- name: start hadoop Jobtracker services |
|
||||||
service: name=hadoop-0.20-mapreduce-jobtrackerha state=started |
|
||||||
delegate_to: ${item} |
|
||||||
with_items: ${groups.hadoop_masters} |
|
||||||
when_set: $jt_result and $jt_result.changed |
|
||||||
|
|
@ -1,4 +1,64 @@ |
|||||||
--- |
--- |
||||||
# Playbook for Hadoop master secondary server |
# Playbook for Hadoop master secondary server |
||||||
|
|
||||||
- include: hadoop_secondary.yml |
|
||||||
|
- name: Install the namenode and jobtracker packages |
||||||
|
yum: name=${item} state=installed |
||||||
|
with_items: |
||||||
|
- hadoop-0.20-mapreduce-jobtrackerha |
||||||
|
- hadoop-hdfs-namenode |
||||||
|
- hadoop-hdfs-zkfc |
||||||
|
- hadoop-0.20-mapreduce-zkfc |
||||||
|
|
||||||
|
- name: Copy the hadoop configuration files |
||||||
|
template: src=roles/common/templates/hadoop_ha_conf/{{ item }}.j2 dest=/etc/hadoop/conf/{{ item }} |
||||||
|
with_items: |
||||||
|
- core-site.xml |
||||||
|
- hadoop-metrics.properties |
||||||
|
- hadoop-metrics2.properties |
||||||
|
- hdfs-site.xml |
||||||
|
- log4j.properties |
||||||
|
- mapred-site.xml |
||||||
|
- slaves |
||||||
|
- ssl-client.xml.example |
||||||
|
- ssl-server.xml.example |
||||||
|
notify: restart hadoopha master services |
||||||
|
|
||||||
|
- name: Create the data directory for the namenode metadata |
||||||
|
file: path={{ item }} owner=hdfs group=hdfs state=directory |
||||||
|
with_items: hadoop.dfs_namenode_name_dir |
||||||
|
|
||||||
|
- name: Create the data directory for the jobtracker ha |
||||||
|
file: path={{ item }} owner=mapred group=mapred state=directory |
||||||
|
with_items: hadoop.mapred_job_tracker_persist_jobstatus_dir |
||||||
|
|
||||||
|
|
||||||
|
- name: Initialize the secodary namenode |
||||||
|
shell: creates=/usr/lib/hadoop/namenode.formatted su - hdfs -c "hadoop namenode -bootstrapStandby"; touch /usr/lib/hadoop/namenode.formatted |
||||||
|
|
||||||
|
- name: start hadoop namenode services |
||||||
|
service: name=hadoop-hdfs-namenode state=started |
||||||
|
|
||||||
|
- name: Initialize the zkfc for namenode |
||||||
|
shell: creates=/usr/lib/hadoop/zkfc.formatted su - hdfs -c "hdfs zkfc -formatZK"; touch /usr/lib/hadoop/zkfc.formatted |
||||||
|
|
||||||
|
- name: start zkfc for namenodes |
||||||
|
service: name=hadoop-hdfs-zkfc state=started |
||||||
|
delegate_to: ${item} |
||||||
|
with_items: groups.hadoop_masters |
||||||
|
|
||||||
|
- name: Give permissions for mapred users |
||||||
|
shell: creates=/usr/lib/hadoop/fs.initialized su - hdfs -c "hadoop fs -chown hdfs:hadoop /"; su - hdfs -c "hadoop fs -chmod 0774 /"; touch /usr/lib/hadoop/namenode.initialized |
||||||
|
|
||||||
|
- name: Initialize the zkfc for jobtracker |
||||||
|
shell: creates=/usr/lib/hadoop/zkfcjob.formatted su - mapred -c "hadoop mrzkfc -formatZK"; touch /usr/lib/hadoop/zkfcjob.formatted |
||||||
|
|
||||||
|
- name: start zkfc for jobtracker |
||||||
|
service: name=hadoop-0.20-mapreduce-zkfc state=started |
||||||
|
delegate_to: ${item} |
||||||
|
with_items: groups.hadoop_masters |
||||||
|
|
||||||
|
- name: start hadoop Jobtracker services |
||||||
|
service: name=hadoop-0.20-mapreduce-jobtrackerha state=started |
||||||
|
delegate_to: ${item} |
||||||
|
with_items: groups.hadoop_masters |
||||||
|
@ -1,4 +1,4 @@ |
|||||||
--- |
--- |
||||||
# Playbook for Hadoop slave servers |
# Playbook for Hadoop slave servers |
||||||
|
|
||||||
- include: slaves.yml tags=slaves,no_ha |
- include: slaves.yml tags=slaves |
||||||
|
@ -1,9 +1,9 @@ |
|||||||
tickTime=2000 |
tickTime=2000 |
||||||
dataDir=/var/lib/zookeeper/ |
dataDir=/var/lib/zookeeper/ |
||||||
clientPort={{ hadoop['zookeeper.clientport'] }} |
clientPort={{ hadoop['zookeeper_clientport'] }} |
||||||
initLimit=5 |
initLimit=5 |
||||||
syncLimit=2 |
syncLimit=2 |
||||||
{% for host in groups['zookeeper_servers'] %} |
{% for host in groups['zookeeper_servers'] %} |
||||||
server.{{ hostvars[host].zoo_id }}={{ host }}:{{ hadoop['zookeeper.leader_port'] }}:{{ hadoop['zookeeper.election_port'] }} |
server.{{ hostvars[host].zoo_id }}={{ host }}:{{ hadoop['zookeeper_leader_port'] }}:{{ hadoop['zookeeper_election_port'] }} |
||||||
{% endfor %} |
{% endfor %} |
||||||
|
|
||||||
|
@ -1,6 +0,0 @@ |
|||||||
--- |
|
||||||
# Vars for Zookeeper |
|
||||||
|
|
||||||
clientport: 2181 |
|
||||||
leader_port: 2888 |
|
||||||
election_port: 3888 |
|
Reference in new issue