[Python][P4][OpenGrok] Automation from P4 to OpenGrok

「因為人會犯錯，所以我們要寫扣。」

敝人是一個很容易犯錯的人，一件事情重複做10次可以錯8次，

所以寫扣是避免自己重複犯同樣錯誤的最好方法。

這次的自動化的任務是定期從P4上拉最新的扣下來，放到OpenGrok上給大家trace code跟code review用，另外還要維護一個首頁搜集每個project的連結。

過去的方法是重複的copy paste, 每次增加一個新的project，就有5個檔案要改。

然而最近發現要sync的project竟然也快20個，每個月又會再增加新的，

雖然原本的方法很難出錯，但我做起來就是錯誤百出。

於是想改成只要在某個設定檔上加一行，然後每件事情就自動完成：

1. 做出新project的webapp, 

2. 在首頁中加入新webapp連結,

3. 更新現有webapp的原始碼

4. index現有的webapp

這裡是幾個設計與遇到的問題：

1. 因為要跟前端JavaScript分享設定檔，設定檔預期使用json格式。

2. 但是用shell script處理json頗麻煩，所以用不是很熟但相對輕鬆的Python來做

3. 過去是用P4的command line tool更新原始碼，所以需要P4的python lib....

幸好，他們真的有這種東西 p4python

安裝有點小曲折，不過裝完就忘了 XDDD

4. OpenGrok的事情就繼續用subprocess + command line tool.

5. 建立新webapp還有一些xml要處理，還好Python都有。

最後測試成功的結果是這樣
當中困擾我最久的就是，
p4python的run_sync預設會將"Up to date"視為exception，
如果沒有catch p4.warnings 後面都不用玩了。
這要仔細看exception_level那段才會發現。

	import os
	import json
	import shutil
	import subprocess
	import sys
	import xml.etree.ElementTree as ET
	from P4 import P4,P4Exception
	P4PORT = "your.p4.server"
	P4USER = "your_account"
	P4PASSWD = "your_password"
	P4CLIENT = "opengrok"
	def sync_p4_sourcecode(depot_list):
	#login p4
	p4 = P4()
	p4.port = P4PORT
	p4.user = P4USER
	p4.password = P4PASSWD
	p4.client = P4CLIENT
	try:
	p4.connect()
	p4.run_login()
	for depot in depot_list:
	try:
	sync_res = p4.run("sync", depot)
	print sync_res
	except P4Exception:
	#"Up to date" goes to warning...
	#To prevent p4python from raise warning as exception,
	#changep4.exception_level to 1 (error only)
	for e in p4.errors:
	print e
	raise
	for e in p4.warnings:
	print e
	p4.run_logout()
	p4.disconnect()
	except:
	print traceback.format_exc()
	return -1
	del p4
	return 0

view raw p42opengrok-1.py hosted with ❤ by GitHub

	#OpenGrok constants
	OPENGROK_INDEX_CMD = ["/usr/opengrok/bin/OpenGrok", "index"]
	OPENGROK_SRC_BASE = ['','home','OPENGROK','src']
	OPENGROK_DATA_BASE = ['','home','OPENGROK','data']
	TOMCAT_BASE = ['','var','lib','tomcat','webapps']
	#Since this script assumes that each depot has it's independent webapp,
	#to create a new webapp for new depot,
	#this function copies setting from a example folder as template,
	#then change value within configuration.xml and web.xml
	WEBAPP_TMPL_FOLDER = os.path.sep.join(TOMCAT_BASE + ['example'])
	def create_opengrok_webapp(context):
	print "creating webapp for %s ..." % context['webapp_name']
	#Check if /home/OPENGROK/<webapp_name> exists, if not, create one.
	#OpenGrok index data goes here.
	if not os.path.isdir(context['opengrok_instance_base']):
	print "mkdir %s" % context['opengrok_instance_base']
	os.mkdir(context['opengrok_instance_base'])
	#Create new webapp
	if not os.path.isdir(context['webapp_base']):
	try:
	shutil.copytree(WEBAPP_TMPL_FOLDER, context['webapp_base'])
	except:
	print traceback.format_exc()
	#Modify config file for new webapp.
	#/<path_to_webapp>/WEB-INF/configuraion.xml
	if os.path.exists(context['conf_dst']):
	root = ET.parse(context['conf_dst'])
	data_root = root.findall(".//void[@property='dataRoot']/string")[0]
	data_root.text = context['opengrok_instance_base']
	source_root = root.findall(".//void[@property='sourceRoot']/string")[0]
	source_root.text = context['opengrok_source_code']
	url_prefix = root.findall(".//void[@property='urlPrefix']/string")[0]
	url_prefix.text = '/' + context['webapp_name'] + '/s?'
	root.write(context['conf_dst'])
	del root
	#/<path_to_webapp>/WEB-INF/web.xml
	webapp_conf = context['conf_dst'].replace('configuration','web')
	root = ET.parse(webapp_conf)
	cp_list = root.findall(".//context-param")
	for cp in cp_list:
	param_name = cp.find('param-name')
	if param_name.text == 'CONFIGURATION':
	param_value = cp.find('param-value')
	param_value.text = context['conf_dst']
	root.write(webapp_conf)
	break
	del root

view raw p42opengrok-2.py hosted with ❤ by GitHub

	def index_opengrok_data(context):
	print "indexing webapp %s ..." % context['webapp_name']
	ENV = {
	"OPENGROK_WEBAPP_CONTEXT" : context['webapp_name'],
	"OPENGROK_INSTANCE_BASE" : context['opengrok_instance_base']
	}
	p = subprocess.Popen(OPENGROK_INDEX_CMD + [context['opengrok_source_code']], stdout=subprocess.PIPE, stderr=subprocess.PIPE,env=ENV)
	while p.poll() is None:
	output, err = p.communicate()
	returncode = p.returncode
	print output
	if err is not None:
	print err
	print returncode
	shutil.copy(context['conf_src'], context['conf_dst'])
	def restart_tomcat():
	TOMCAT_RESTART_CMD = ["systemctl", "restart", "tomcat"]
	p = subprocess.Popen(TOMCAT_RESTART_CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	while p.poll() is None:
	output, err = p.communicate()
	returncode = p.returncode
	print output
	if err is not None:
	print err
	print returncode

view raw p42opengrok-3.py hosted with ❤ by GitHub

	depot_list = None
	with open('depot_list.json') as depot_list_file:
	depot_list = json.load(depot_list_file)
	sync_p4_sourcecode(depot_list)
	for depot in depot_list:
	tmp_path = filter(None, depot.split('/'))
	context = dict()
	#Name webapp after depot folder name.
	context['webapp_name'] = tmp_path[len(tmp_path) -1 ]
	#Source codes are here
	context['opengrok_source_code'] = os.path.sep.join(OPENGROK_SRC_BASE)
	#OpenGrok data are here
	context['opengrok_instance_base'] = os.path.sep.join(OPENGROK_DATA_BASE + [context['webapp_name']])
	#webapp path
	context['webapp_base'] = os.path.sep.join(TOMCAT_BASE + [context['webapp_name']])
	#conf file generated after indexing
	context['conf_src'] = os.path.sep.join(OPENGROK_DATA_BASE + [context['webapp_name'], 'etc', 'configuration.xml'])
	#conf file in webappp, to be updated with context['conf_src']
	context['conf_dst'] = os.path.sep.join(TOMCAT_BASE + [context['webapp_name'], 'WEB-INF', 'configuration.xml'])
	if not os.path.isdir(context['webapp_base']) or not os.path.isdir(context['opengrok_instance_base']):
	print "%s does not exist." % context['webapp_base']
	create_opengrok_webapp(context)
	index_opengrok_data(context)
	restart_tomcat()

view raw p42opengrok-4.py hosted with ❤ by GitHub