User:Nskaggs
Jump to navigation
Jump to search
wmcs-netbox-list.py
An enhanced fork of Arturo's netbox-list.py with more fields gathered. Fetch & list WMCS server info from Netbox and generate a CSV (to later import into a spreadsheet):
wmcs-netbox-list.py |
---|
The following content has been placed in a collapsed box for improved usability. |
#!/usr/bin/env python3
import requests
import argparse
class Config:
""" configuration for this script.
"""
def __init__(self, api_token, netbox_url):
self.api_token = api_token
if not netbox_url:
self.netbox_url = "https://netbox.wikimedia.org"
else:
self.netbox_url = netbox_url
def generate_netbox_link(config, id):
return "{}/dcim/devices/{}/".format(config.netbox_url, id)
def print_result_header():
print(
"name, purchase date, ticket, status, manufacturer, model, serial, asset tag, rack, position, IPv4, vlan, site, netbox link"
)
def print_result(config, result):
netbox_link = generate_netbox_link(config, result["id"])
if not result["primary_ip4"]:
result["primary_ip4"] = {}
result["primary_ip4"]["address"] = "None"
vlan = "None"
else:
vlan = request_query_vlan(config, result)
if not result["rack"]:
result["rack"] = {}
result["rack"]["name"] = "None"
print(
"{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}".format(
result["name"],
result["custom_fields"]["purchase_date"],
result["custom_fields"]["ticket"],
result["status"]["value"],
result["device_type"]["manufacturer"]["name"],
result["device_type"]["model"],
result["serial"],
result["asset_tag"],
result["rack"]["name"],
result["position"],
result["primary_ip4"]["address"],
vlan,
result["site"]["slug"],
netbox_link,
)
)
def request_query_list(config, query):
request_url = "{}/api/dcim/devices/?q={}".format(config.netbox_url, query)
request_headers = {"Authorization": "Token {}".format(config.api_token)}
return requests.get(url=request_url, headers=request_headers)
def request_query_vlan(config, result):
ipaddr = result["primary_ip4"]["id"]
request_url = "{}/api/ipam/ip-addresses/{}".format(config.netbox_url, ipaddr)
request_headers = {"Authorization": "Token {}".format(config.api_token)}
ip_result = requests.get(url=request_url, headers=request_headers)
if not ip_result.json()["assigned_object_id"]:
return "None"
interface = ip_result.json()["assigned_object_id"]
request_url = "{}/api/dcim/interfaces/{}".format(config.netbox_url, interface)
request_headers = {"Authorization": "Token {}".format(config.api_token)}
interface_result = requests.get(url=request_url, headers=request_headers)
if not interface_result.json()["connected_endpoint"]:
return "None"
endpoint = interface_result.json()["connected_endpoint"]["id"]
request_url = "{}/api/dcim/interfaces/{}".format(config.netbox_url, endpoint)
request_headers = {"Authorization": "Token {}".format(config.api_token)}
endpoint_result = requests.get(url=request_url, headers=request_headers)
if endpoint_result.json()["untagged_vlan"]:
return endpoint_result.json()["untagged_vlan"]["name"]
else:
return "None"
def main():
parser = argparse.ArgumentParser(
description="Utility to fetch and list WMCS servers from Netbox"
)
parser.add_argument("--token", action="store", help="Netbox API token")
parser.add_argument("--url", action="store", help="Netbox URL")
args = parser.parse_args()
config = Config(args.token, args.url)
print_result_header()
r = request_query_list(config, "cloud")
for result in r.json()["results"]:
print_result(config, result)
r = request_query_list(config, "lab")
for result in r.json()["results"]:
print_result(config, result)
r = request_query_list(config, "labtest")
for result in r.json()["results"]:
print_result(config, result)
if __name__ == "__main__":
main()
|
The above content has been placed in a collapsed box for improved usability. |
wmcs-phab.py
Proof of concept to pull ticket information from phabricator to csv
wmcs-phab.py |
---|
The following content has been placed in a collapsed box for improved usability. |
#!/usr/bin/env python3
import argparse
from phabricator import Phabricator
import time
class Config:
""" configuration for this script.
"""
def __init__(self, api_token, phab_url, project):
self.api_token = api_token
if not phab_url:
self.phab_url = "https://phabricator.wikimedia.org/api/"
else:
self.phab_url = phab_url
if not project:
#cloud services kanban
self.project = "2774"
else:
self.project = project
self.phab = Phabricator(host=self.phab_url, token=self.api_token)
def print_result_header():
print(
"id, dateCreated, dateModified, status, priority, author, owner, title"
)
def print_result(result):
print(
"{}, {}, {}, {}, {}, {}, {}, {}".format(
result["id"],
time.strftime("%Y-%m-%d", time.gmtime(result["created"])),
time.strftime("%Y-%m-%d", time.gmtime(result["modified"])),
#result["created"],
#result["modified"],
result["status"],
result["priority"],
result["author"],
result["owner"],
result["title"]
)
)
def open_phab_tasks(config):
project = config.phab.project.query(ids=[config.project])
p_pid = list(project['data'].keys())[0]
tasks = config.phab.maniphest.query(projectPHIDs=[p_pid],status='status-open')
results = []
for key, task in tasks.items():
a_pid = task['authorPHID']
o_pid = task['ownerPHID']
author = config.phab.user.query(phids=[a_pid])[0]["userName"]
if config.phab.user.query(phids=[o_pid]):
owner = config.phab.user.query(phids=[o_pid])[0]["userName"]
else:
owner = "None"
result = {}
result['id'] = task['id']
result['created'] = int(task['dateCreated'])
result['modified'] = int(task['dateModified'])
result['status'] = task['status']
result['priority'] = task['priority']
result['author'] = author
result['owner'] = owner
result['title'] = task['title'].replace(',','')
results.append(result)
return results
def main():
parser = argparse.ArgumentParser(
description="Utility to information from Phabricator"
)
parser.add_argument("--token", action="store", help="Phabricator API token")
parser.add_argument("--url", action="store", help="Phabricator URL")
parser.add_argument("--project", action="store", help="Project ID")
args = parser.parse_args()
config = Config(args.token, args.url, args.project)
print_result_header()
results = open_phab_tasks(config)
for result in results:
print_result(result)
if __name__ == "__main__":
main()
|
The above content has been placed in a collapsed box for improved usability. |
wmcs-openstack-usage.py
Quick script used to help generate raw data for resource usage of WMCS cloud
wmcs-openstack-usage.py |
---|
The following content has been placed in a collapsed box for improved usability. |
#!/usr/bin/env python3
import io
import subprocess
def main():
startdate = '2021-12-01'
enddate = '2022-01-01'
f = open('usage.csv', 'w')
f.write('Project, Instances, VCPU, MEM (GB), DISK (GB)\n')
# get list of projects
output = subprocess.run(['sudo', 'wmcs-openstack','project','list', '-f', 'value', '-c', 'ID'], stdout=subprocess.PIPE).stdout.decode('utf-8')
projects = output.split('\n')[:-1]
for project in projects:
# gather limit stats
output = subprocess.run(['sudo', 'wmcs-openstack','limits','show', '--project', project, '--absolute', '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
limits = output.split('\n')
# parse out resource usage and limits
for line in limits:
attr = line.split(' ')[0]
if attr == 'totalInstancesUsed':
instances = line.split(' ')[1]
elif attr == 'totalCoresUsed':
cpu = line.split(' ')[1]
elif attr == 'totalRAMUsed':
ram = line.split(' ')[1]
# Convert to GB
ram = str(int(ram) / 1024)
#parse stats and output as csv
f.write(f'{project},{instances},{cpu},{ram},')
# in order to get disk usage, we need to check other calls
# gather disk used (cinder)
output = subprocess.run(['sudo', 'wmcs-openstack','resource','usage', 'show', '--os-placement-api-version', '1.9', project, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
resources = output.split('\n')
for line in resources:
attr = line.split(' ')[0]
if attr == 'DISK_GB':
disk_gb = line.split(' ')[1]
f.write(f'{disk_gb}\n')
f.close()
if __name__ == "__main__":
main()
|
The above content has been placed in a collapsed box for improved usability. |
wmcs-openstack-resources.py
Quick script used to demonstrate raw data on resource usage of WMCS cloud
wmcs-openstack-usage.py |
---|
The following content has been placed in a collapsed box for improved usability. |
#!/usr/bin/env python3
import io
import subprocess
def main():
startdate = '2021-08-21'
enddate = '2021-09-21'
fu = open('usage.csv', 'w')
fu.write(f'Project,CPU Hours,Disk GB-Hours,RAM MB-Hours,Servers\n')
fl = open('limits.csv', 'w')
fl.write('Project, maxInstances, Instances, maxCPU, CPU, maxRAM, RAM (MB))\n')
fq = open('quota.csv', 'w')
fq.write('Project, gigabytes\n')
fr = open('resource.csv', 'w')
fr.write('Project, VCPU, MEMORY_MD, DISK_G\n')
# get list of projects
output = subprocess.run(['sudo', 'wmcs-openstack','project','list', '-f', 'value', '-c', 'ID'], stdout=subprocess.PIPE).stdout.decode('utf-8')
projects = output.split('\n')[:-1]
for project in projects:
# gather usage stats
output = subprocess.run(['sudo', 'wmcs-openstack','usage','show', '--project', project, '--print-empty', '-f', 'value', '--start', startdate, '--end', enddate], stdout=subprocess.PIPE).stdout.decode('utf-8')
usage = output.split('\n')[:-1]
#parse stats and output as csv
fu.write(f'{project},{usage[0]},{usage[1]},{usage[2]},{usage[3]}\n')
# gather limit stats
output = subprocess.run(['sudo', 'wmcs-openstack','limits','show', '--project', project, '--absolute', '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
limits = output.split('\n')
# parse out resource usage and limits
for line in limits:
attr = line.split(' ')[0]
if attr == 'maxTotalInstances':
maxinstances = line.split(' ')[1]
elif attr == 'totalInstancesUsed':
instances = line.split(' ')[1]
elif attr == 'maxTotalCores':
maxcpu = line.split(' ')[1]
elif attr == 'totalCoresUsed':
cpu = line.split(' ')[1]
elif attr == 'maxTotalRAMSize':
maxram = line.split(' ')[1]
elif attr == 'totalRAMUsed':
ram = line.split(' ')[1]
#parse stats and output as csv
fl.write(f'{project},{maxinstances},{instances},{maxcpu},{cpu},{maxram},{ram},\n')
# in order to get disk usage, we need to check other calls
# gather maxdisk (cinder)
ps = subprocess.run(['sudo', 'wmcs-openstack','quota','show', project, '-f', 'table'], stdout=subprocess.PIPE)
pipe = subprocess.run(['grep', '| gigabytes |'], input=ps.stdout, stdout=subprocess.PIPE)
output = subprocess.run(['cut', '-d', '|', '-f', '3'], input=pipe.stdout, stdout=subprocess.PIPE).stdout.decode('utf-8')
diskquota = output.strip()
fq.write(f'{project},{diskquota},\n')
# gather disk used (cinder)
output = subprocess.run(['sudo', 'wmcs-openstack','resource','usage', 'show', '--os-placement-api-version', '1.9', project, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
resources = output.split('\n')
for line in resources:
attr = line.split(' ')[0]
if attr == 'VCPU':
vcpu = line.split(' ')[1]
elif attr == 'MEMORY_MB':
memory_mb = line.split(' ')[1]
elif attr == 'DISK_GB':
disk_gb = line.split(' ')[1]
fr.write(f'{project},{vcpu},{memory_mb},{disk_gb}\n')
fu.close()
fl.close()
fq.close()
fr.close()
fh = open('hosts.csv', 'w')
fh.write(f'Host,Summary,CPU,Memory MB,Disk GB\n')
fp = open('projects.csv', 'w')
fp.write(f'Project,Host,CPU,Memory MB,Disk GB\n')
# get list of hosts and show
output = subprocess.run(['sudo', 'wmcs-openstack','host','list', '-f', 'value', '-c', 'Host Name', '--zone', 'nova'], stdout=subprocess.PIPE).stdout.decode('utf-8')
hosts = output.split('\n')[:-1]
# seperate host totals from project stats
for host in hosts:
output = subprocess.run(['sudo', 'wmcs-openstack','host','show', host, '-f', 'value'], stdout=subprocess.PIPE).stdout.decode('utf-8')
stats = output.split('\n')[:-1]
host_lines = stats[0:3]
project_lines = stats[3:]
# write out host info
for line in host_lines:
entry = line.split(' ')
fh.write(f'{entry[0]},{entry[1]},{entry[2]},{entry[3]},{entry[4]}\n')
# write out project info
for line in project_lines:
entry = line.split(' ')
fp.write(f'{entry[1]},{entry[0]},{entry[2]},{entry[3]},{entry[4]}\n')
fh.close()
fp.close()
main()
|
The above content has been placed in a collapsed box for improved usability. |