Python – RegEx examples

Searching for text in multiple files within a directory OR one file in a directory

This script searches the yum directory for all the repo files and searches for the URLs each repo goes to.

Search for text in multiple files

src_dict = ("/etc/yum.repos.d/") #Specify base directory
pattern = re.compile ('http\S+') #CPatter to search for

for yum_files in os.listdir(src_dict): # obtain list of files in directory
    files = os.path.join(src_dict, yum_files) #join the full path with the names of the files.
    strng = open(files) #We need to open the files
    for lines in strng.readlines(): #We then need to read the files
        if re.search(pattern, lines): #If we find the pattern we are looking for
            print re.split(r'=', lines)[1] #We split using as a delimeter the = sign.
            #INSERT WHATEVER CODE YOU WANT

Search for text in one file in a directory

file = 'file.csv'
drc = '/home/gmastrokostas/PycharmProjects/learning'
f = open( os.path.join(drc, file) )
for lines in f.readlines():
    searpatt = re.search(pattern, lines)
    if re.search(pattern,lines):
        print (searpatt.group())

Change one specific extension type of files in a directory that contains multiple types of extensions

import os
import re

src_drct='/home/gmastrokostas/tmp'

for files in os.listdir(src_drct):

    if files.endswith('.txt'):
        oldF = os.path.join(src_drct, files)
        #midF = re.split(r'\.', files)#This works too.
        midF = os.path.splitext(files) #It creates a list and splits the name of the file from the extension.
        newF = oldF.replace('.txt', '.py')#Simple replacement.
        out = os.rename(oldF, newF)

Search for specific types of files and rename the files or part of the files

The script finds files with a specific file extension and renames parts of the files it found.

import os
import re
 
src_drct='/home/gmastrokostas/tmp'
 
for files in os.listdir(src_drct):
 
    if files.endswith('.txt'): #Select all files with the .txt ext
        oldF = os.path.join(src_drct, files) #Join full path with files found
        midF = re.split(r'\_', files) #split files that contain underscore
        newF = oldF.replace('_01012015', '_01012014') #create the replacement var
        out = os.rename(oldF, newF) #start renaming.

Replace a string in multiple files with in a given directory

The script below will search for a string in files with in a directory and subdirectories. Before it starts replacing any text it will create a backup of the original file(s). Note the script does not into account possible duplicate files when copying the files in /tmp.

import re
import os
import shutil

drc = '/root/tmp'
backup = '/tmp'
pattern = re.compile('PYTHON')
oldstr = 'PYTHON'
newstr = 'Python'

for dirpath, dirname, filename in os.walk(drc):#Getting a list of the full paths of files
    for fname in filename:
        path = os.path.join(dirpath, fname) #Joining dirpath and filenames
        strg = open(path).read() #Opening the files for reading only
        if re.search(pattern, strg):#If we find the pattern ....
            #print path, strg
            shutil.copy2(path, backup) #we will create a backup of it
            strg = strg.replace(oldstr, newstr) #We will create the replacement condistion
            f = open(path, 'w') #We open the files with the WRITE option
            f.write(strg) # We are writing the the changes to the files
            f.close() #Closing the files

Replace a string in One file with in a given directory

 
oldstr = 'Time'
newstr = 'TIME'
file_path = '/home/gmastrokostas/PycharmProjects/learning/file.csv'
with open(file_path, 'r') as f:
   fread = f.read()
   strg  = fread.replace(oldstr, newstr)
   fopen = open(file_path, 'w')
   fopen.write(strg)
f.close()

Write text from file temp1.txt to named.txt (use the ‘a’ option if you want to append instead)

 
#!/usr/bin/python
import os
import re
with open("named.txt", "wt") as fout:
    with open("templ.txt", "rt") as fin:
        for line in fin:
            fout.write(line)

Search for a string in a file and display the Result

 
import re
text="XXX"
file ='templ.txt'
for line in open(file):
    if text in line:
        print line

Search for a string by using compile in a file and display the Result

 
import re
text = "XXX"
pattern = re.compile(text)
file = 'templ.txt'
of = open(file)
for line in of.readlines():
    if re.search(pattern,  line):
        print line

Append the contents of a file to another file

import re
import os
fo = open("templ.txt","r")
fi = open("named.txt","a")
for line in fo:
    fi.write(line)
    #print line
fi.close()
fo.close()

Isolating text field from a variable

text = '11:47:55.045 -T- MFS_TEST10 Received FIX Message 3  8=FIX.4.2|9=146|35=D|34=3|49=MFS_TEST10|52=20120726-15:47:55|56=CCG|115=YYYC|11=AAA 0981/07262012|54=1|38=100000|55=ACL|40=1|47=A|60=20120726-15:47:55|21=1|207=N|10=044|'
print re.split(r'\s', text)[2]
Output
MFS_TEST10

Isolating text fields from a file.

 import re
#!/usr/bin/python
import re
import os
import fnmatch
fo = open("templ.txt", "r")
for line in fo:
        print line
        print re.split(r'\s', line, re.I|re.M)[1]
Provided the text with in the file was "Road 123 Str NY USA 11214"
Output 123

Searching for specific type files.

 

filenames = ['httpd.conf', 'samba.conf', 'header.h', 'socket.net']

[loop for loop in filenames if loop.endswith(('.h'))]
for loop in filenames:
    if loop.endswith('.h') or loop.startswith('httpd'):
        print loop
Output
httpd.conf
header.h

Searching through a List

 
#Great use with os.dirlist
addresses = [
    '5412 N CLARK ST',
    '1060 W ADDISON ST',
    '1039 W GRANVILLE AVE',
    '2122 N CLARK ST',
    '4802 N BROADWAY',
]

for loop in addresses:
    if fnmatch.fnmatchcase(loop, '* ST'):
        print loop
Output
5412 N CLARK ST
1060 W ADDISON ST
 2122 N CLARK ST

Print specific fields from a string

 
text = 'Today is 11/27/2012. PyCon starts 3/13/2013.'
pattern = re.compile('Today\s\w+\s\d+\W\d+\W\d+', re.IGNORECASE)
searpatt = re.search(pattern, text)
print (searpatt.group())

Output

Today is 11/27/2012

Replace specific fields of text

print text
#!/usr/bin/python
import re

text = 'Today is 11/27/2012. PyCon starts 3/13/2013.'
newt = "Yesterday"
pattern = re.compile('Today\s\w+\s\d+\W\d+\W\d+', re.IGNORECASE)
searpatt = re.search(pattern, text)
print re.sub(pattern, newt, text)

Output

Yesterday. PyCon starts 3/13/2013.

Print a specific element of the split you created

#!/usr/bin/python
import re

text = 'Today is 11/27/2012. PyCon starts 3/13/2013.'
newt = "Yesterday"
pattern = re.compile('Today\s\w+\s\d+\W\d+\W\d+', re.IGNORECASE)
searpatt = re.search(pattern, text)
print text
print re.split(pattern, text )[1]

PyCon starts 3/13/2013.

Replace strings in a file

#!/usr/bin/python
import os
import re
text_ip = "#listen_addresses = 'localhost'"
        subtext_ip = "listen_addresses = '10.0.0.27'"
        text_port = "#port = 5432"
        subtext_port = "port = 5432"

        s = open("/home/postgres/postgres94/postgresql.conf").read()
        s = s.replace(text_ip, subtext_ip)
        s = s.replace(text_port, subtext_port)
        f = open("/home/postgres/postgres94/postgresql.conf", 'w')
        f.write(s)
        f.close()
Share Button

Python/Postgresql – Query Database and present data in a plot

How to query a database using a Python script and present the data on a plot.

import pandas.io.data
import matplotlib.pyplot as plt
import pandas as pd
from pandas import DataFrame
import datetime
import pandas.io.data
import psycopg2
import re
import os


conn = psycopg2.connect("host='172.31.98.161' dbname='servers' user='seeker'")
cur = conn.cursor() #Create the cursor
cur.execute("select total_ram_raw, used_ram_raw, time_captured  from server_perf  where hostname='localhost.localdomain'")
rows = cur.fetchall()
conn.close()

df = pd.DataFrame(rows, columns=['TotalRam','UsedRAM', 'Time'])

df.to_csv("file.csv")
df.TotalRam = df.TotalRam.astype(float)
df.UsedRAM  = df.UsedRAM.astype(float)

df = pd.read_csv('file.csv', index_col='Time')
df[['TotalRam', 'UsedRAM']].plot()
plt.show()
Share Button

Python – Replace a string in multiple files with in a directory

The script below will search for a string in files with in a directory and subdirectories. Before it starts replacing any text it will create a backup of the original file(s).

import re
import os
import shutil

drc = '/root/tmp'
backup = '/tmp'
pattern = re.compile('PYTHON')
oldstr = 'PYTHON'
newstr = 'Python'

for dirpath, dirname, filename in os.walk(drc):#Getting a list of the full paths of files
    for fname in filename:
        path = os.path.join(dirpath, fname) #Joining dirpath and filenames
        strg = open(path).read() #Opening the files for reading only
        if re.search(pattern, strg):#If we find the pattern ....
            #print path, strg
            shutil.copy2(path, backup) #we will create a backup of it
            strg = strg.replace(oldstr, newstr) #We will create the replacement condistion
            f = open(path, 'w') #We open the files with the WRITE option
            f.write(strg) # We are writing the the changes to the files
            f.close() #Closing the files

Share Button

Python – Search for specific types of files and rename the files or part of the files

The script finds files with a specific file extension and renames parts of the files it found.

import os
import re
 
src_drct='/home/gmastrokostas/tmp'
 
for files in os.listdir(src_drct):
 
    if files.endswith('.txt'): #Select all files with the .txt ext
        oldF = os.path.join(src_drct, files) #Join full path with files found
        midF = re.split(r'\_', files) #split files that contain underscore
        newF = oldF.replace('_01012015', '_01012014') #create the replacement var
        out = os.rename(oldF, newF) #start renaming.
Share Button

Python – Change one specific extension type of files in a directory that contains multiple types of extensions

import os
import re

src_drct='/home/gmastrokostas/tmp'

for files in os.listdir(src_drct):

    if files.endswith('.txt'):
        oldF = os.path.join(src_drct, files)
        #midF = re.split(r'\.', files)#This works too.
        midF = os.path.splitext(files) #It creates a list and splits the name of the file from the extension.
        newF = oldF.replace('.txt', '.py')#Simple replacement.
        out = os.rename(oldF, newF)

Share Button

Python – Parse URLs from files in directory to check if links are up/down before executing yum on Centos.

This script parses the files in the yum repo directory and checks if all URLS are up before proceeding to execute yum. It creates two lists to do a sanity check. One list stores the number of URLs found and the second list stores the number of links that have been found to be working. If those two numbers do not match the execution of YUM is aborted. Please note, this script will work only with custom made yum repos.

import os
import subprocess
import re
import urllib2
import os.path


def check_Software():
    path = '/usr/bin/gcc'
    if os.path.isfile(path):
        pass
    else:
        try:
            subprocess.call(["yum", "install", "-y", "gcc"])
            if not os.patch.isfile(path):
                print "Sending email"

        except OSError as er:
            print er

def yum_link_Search():
    #These two lists are being used as control. The status list is used to append succesful HTTP connections
    #for each of the URLs captured from the files in the YUM directory. The latter is stored in the files_list list.
    #The number (amount of elements) is being compared. A success condition is when each site returns an alive code of
    #200. This tells us that all sites are up and running. View lines between 52 and 60
    status_list = []
    files_list = []
    src_dict = ("/etc/yum.repos.d/") #Specify base directory
    pattern = re.compile ('http\S+') #CPatter to search for

    for yum_files in os.listdir(src_dict): # obtain list of files in directory
        files = os.path.join(src_dict, yum_files) #join the full path with the names of the files.
        files_list.append(yum_files)
        strng = open(files) #We need to open the files
        for lines in strng.readlines(): #We then need to read the files
            if re.search(pattern, lines): #If we find the pattern we are looking for
                print re.split(r'=', lines)[1] #We split using as a delimeter the = sign.
                try:

                    status_link = urllib2.urlopen(re.split(r'=', lines)[1]).getcode()
                    #if  status_link == 200: #Check if the site is up
                    status_list.append(status_link)#View notes starting on line 29 for these conditionals
                    print status_list
                    print files
                    if len(files_list) > len(status_list): #View notes starting on line 29 for these conditionals
                        print "Not all Links are up - ABORTING"
                        exit();
                    if len(files_list) < len(status_list):
                        print "Not all Links are up - ABORTING"
                        exit();
                    elif len(files_list) == len(status_list):
                        print "Will run yum installer" #here we can call other functions since everything seems to be working fine.
                        #call the yum function here
                except StandardError as er:#Here we are capturing errors of other codes.
                    print "Site down"
                    print status_link



yum_link_Search()
Share Button

Python – Searching for text in multiple files within a directory OR one file in a directory

This script searches the yum directory for all the repo files and searches for the URLs each repo goes to.

Search for text in multiple files

src_dict = ("/etc/yum.repos.d/") #Specify base directory
pattern = re.compile ('http\S+') #CPatter to search for

for yum_files in os.listdir(src_dict): # obtain list of files in directory
    files = os.path.join(src_dict, yum_files) #join the full path with the names of the files.
    strng = open(files) #We need to open the files
    for lines in strng.readlines(): #We then need to read the files
        if re.search(pattern, lines): #If we find the pattern we are looking for
            print re.split(r'=', lines)[1] #We split using as a delimeter the = sign.
            #INSERT WHATEVER CODE YOU WANT

Search for text in one file in a directory

drc = '/home/gmastrokostas/PycharmProjects/learning'
f = open( os.path.join(drc, file) )
for lines in f.readlines():
    searpatt = re.search(pattern, lines)
    if re.search(pattern,lines):
        print (searpatt.group())
Share Button

Python regex examples

Write text from file temp1.txt to named.txt (use the ‘a’ option if you want to append instead)

 
#!/usr/bin/python
import os
import re
with open("named.txt", "wt") as fout:
    with open("templ.txt", "rt") as fin:
        for line in fin:
            fout.write(line)

Search for a string in a file and display the Result

 
import re
text="XXX"
file ='templ.txt'
for line in open(file):
    if text in line:
        print line

Search for a string by using compile in a file and display the Result

 
import re
text = "XXX"
pattern = re.compile(text)
file = 'templ.txt'
of = open(file)
for line in of.readlines():
    if re.search(pattern,  line):
        print line

Append the contents of a file to another file

import re
import os
fo = open("templ.txt","r")
fi = open("named.txt","a")
for line in fo:
    fi.write(line)
    #print line
fi.close()
fo.close()

Isolating text field from a variable

text = '11:47:55.045 -T- MFS_TEST10 Received FIX Message 3  8=FIX.4.2|9=146|35=D|34=3|49=MFS_TEST10|52=20120726-15:47:55|56=CCG|115=YYYC|11=AAA 0981/07262012|54=1|38=100000|55=ACL|40=1|47=A|60=20120726-15:47:55|21=1|207=N|10=044|'
print re.split(r'\s', text)[2]
Output
MFS_TEST10

Isolating text fields from a file.

 import re
#!/usr/bin/python
import re
import os
import fnmatch
fo = open("templ.txt", "r")
for line in fo:
        print line
        print re.split(r'\s', line, re.I|re.M)[1]
Provided the text with in the file was "Road 123 Str NY USA 11214"
Output 123

Searching for specific type files.

 

filenames = ['httpd.conf', 'samba.conf', 'header.h', 'socket.net']

[loop for loop in filenames if loop.endswith(('.h'))]
for loop in filenames:
    if loop.endswith('.h') or loop.startswith('httpd'):
        print loop
Output
httpd.conf
header.h

Searching through a List

 
#Great use with os.dirlist
addresses = [
    '5412 N CLARK ST',
    '1060 W ADDISON ST',
    '1039 W GRANVILLE AVE',
    '2122 N CLARK ST',
    '4802 N BROADWAY',
]

for loop in addresses:
    if fnmatch.fnmatchcase(loop, '* ST'):
        print loop
Output
5412 N CLARK ST
1060 W ADDISON ST
 2122 N CLARK ST

Print specific fields from a string

 
text = 'Today is 11/27/2012. PyCon starts 3/13/2013.'
pattern = re.compile('Today\s\w+\s\d+\W\d+\W\d+', re.IGNORECASE)
searpatt = re.search(pattern, text)
print (searpatt.group())

Output

Today is 11/27/2012

Replace specific fields of text

print text
#!/usr/bin/python
import re

text = 'Today is 11/27/2012. PyCon starts 3/13/2013.'
newt = "Yesterday"
pattern = re.compile('Today\s\w+\s\d+\W\d+\W\d+', re.IGNORECASE)
searpatt = re.search(pattern, text)
print re.sub(pattern, newt, text)

Output

Yesterday. PyCon starts 3/13/2013.

Print a specific element of the split you created

#!/usr/bin/python
import re

text = 'Today is 11/27/2012. PyCon starts 3/13/2013.'
newt = "Yesterday"
pattern = re.compile('Today\s\w+\s\d+\W\d+\W\d+', re.IGNORECASE)
searpatt = re.search(pattern, text)
print text
print re.split(pattern, text )[1]

PyCon starts 3/13/2013.

Replace strings in a file

#!/usr/bin/python
import os
import re
text_ip = "#listen_addresses = 'localhost'"
        subtext_ip = "listen_addresses = '10.0.0.27'"
        text_port = "#port = 5432"
        subtext_port = "port = 5432"

        s = open("/home/postgres/postgres94/postgresql.conf").read()
        s = s.replace(text_ip, subtext_ip)
        s = s.replace(text_port, subtext_port)
        f = open("/home/postgres/postgres94/postgresql.conf", 'w')
        f.write(s)
        f.close()
Share Button

Python /PostgresqSQL 9.4 – Server Performance Data Capture V.1

These scripts gather static and dynamic information from servers and insert that data into a PostgresSQL database. The static information is information that unless a major upgrade takes place it never changes. They Dynamic data is performance data of the servers. The purpose of the static data is to be able to query for dynamic information which being inserted to the database every X amount of minutes via cron.

These scripts work only on Linux/Unix based machines.

The static information for the remote servers are as follows :

  • Hostname
  • IP
  • Operating System
  • Operating System Version
  • Total Amount of RAM
  • Total Amount of HD space
  • CPU Type
  • MAC ADDRESS

The dynamic information for the remote servers are as follows :

  • Memory Usage
  • Storage Usage
  • Swap Usage
  • Time the data was captured

The static.py script will need to be run only one time on the remote servers or when a major upgrade occurs that might change configuration regarding RAM, Partitions, IP, Operating System (even an upgrade), CPU, NIC replacement.

The dynamic.py script will be run on the remote servers via cron. It is the script that captures information which is being constantly changed, like memory, storage, swap usage. All this data is sent for insertion to the remote database. The script executes it self via cron and then sent to be inserted into a PostgreSQL database.

In both scripts data is entered into a dictionary and then a connection to the database is created in order to insert the data.

The install.py script is not required to be run but it does several things:

  • Installs the PostgresSQL 9.4 database.
  • Creates the database and the role assigned to the database.
  • Creates the Tables needed
  • Configures the postgres.conf file to listen to a specific IP address and port.

However, if you wish to manually setup the database, the SQL script is provided at the bottom of this page. The script is called server_perf.sql

The static.py Script.

#!/usr/bin/python
import psutil
import os
import math
import platform
import subprocess
import socket
import psycopg2
import netifaces as ni

from decimal import *

#Used by the convert values function. It is needed to provide human readable data for the RAM amount produced.
factor = 1024

def Host_name():
    #Get the hostname
    hostname = socket.gethostname()
    return hostname

def IP_address():
    #Get the IP for ETH0
    ni.ifaddresses('eth0')
    ip = ni.ifaddresses('eth0')[2][0]['addr']
    return ip

def OS_make():
    #CPU INFORMATION. Provides the OS make (Windows, Ubunty, RH, Centos, etc)
    #cpu_version = platform.dist()[0]+" "+platform.dist()[1]
    cpu_version = platform.dist()[0]
    return  cpu_version

def OS_Version():
    #Provides the version of the OS.
    os_version = platform.dist()[1]
    return os_version

def Virtual_memory_total():
    #Provides total amount of RAM
    cvr_info = psutil.virtual_memory().total
    return round(cvr_info, 2)

def convert_values():
    #Converts the KB to GB and presents it as such.
    cvr_info = Virtual_memory_total()
    i = int(math.log(cvr_info)/math.log(factor))
    result = float(cvr_info)/factor**i
    return result

def get_processor_info():
    #Gets the verbose info about the CPU (AMD FX(tm)-6350)
    if platform.system() == "Windows":
        return platform.processor()
    elif platform.system() == "Darwin":
        return subprocess.check_output(['/usr/sbin/sysctl', "-n", "machdep.cpu.brand_string"]).strip()
    elif platform.system() == "Linux":
        command = "cat /proc/cpuinfo | grep 'model name' | head -1 | awk -F: '{print $2}'"
        return subprocess.check_output(command, shell=True).strip()
    return ""

#Adjusting precision and then appending to list information taken from Virtual_memory_total function
mem_frmt =   "{0:.2f}".format(convert_values())

#Create the Database PostgreSQL 9.4 connection.
conn = psycopg2.connect("host='10.0.0.41' dbname='serverinfo' user='gmastrokostas'")
cur = conn.cursor() #Create the cursor
#Create a Dictionary to pass the value of each function.
server_info = {'hostname': Host_name(), 'IP': IP_address(), 'OS':OS_make(), 'OS_V':OS_Version(), 'RAM':mem_frmt, 'CPU':get_processor_info()}
cur.execute("INSERT INTO servers(hostname, IP, OS, OS_Version, RAM, CPU) VALUES('%s','%s','%s','%s','%s','%s')" % (Host_name(),IP_address(), OS_make(), OS_Version(), mem_frmt, get_processor_info() ))

#If this is not present the changes will not get commited.
conn.commit()

The Dynamic.py script

import psutil
import os
import math
import platform
import subprocess
import socket
import psycopg2
import netifaces as ni
 
def Host_name():
    #Get the hostname
    hostname = socket.gethostname()
    return hostname
 
def IP_address():
    #Get the IP for ETH0
    ni.ifaddresses('eth0')
    ip = ni.ifaddresses('eth0')[2][0]['addr']
    return ip
 
factor = 1024
def Virtual_memory_use():
    #Provides total amount of RAM
    cvr_info = psutil.virtual_memory().used
    #cvr_info_ram = Virtual_memory_use()
    i = int(math.log(cvr_info)/math.log(factor))
    result_ram = float(cvr_info)/factor**i
    return round(result_ram)
memory_used_frmt =   "{0:.2f}".format(Virtual_memory_use())
 
def Partition_root_Usedspace():
    #Provides total amount of RAM
    cvr_info = psutil.disk_usage('/').used
    #cvr_info_ram = Virtual_memory_use()
    i = int(math.log(cvr_info)/math.log(factor))
    result_ram = float(cvr_info)/factor**i
    return round(result_ram)
hd_used_frmt =   "{0:.2f}".format(Partition_root_Usedspace())
 
 
def CPU_general_usage():
    #CPU INFORMATION.
    cpu_prec_usage = psutil.cpu_percent(interval=0)
    return cpu_prec_usage
 
def Swap_memory_usage():
    #SWAP SIZE USAGE
    swap_mem_full_num = psutil.swap_memory().used
    return swap_mem_full_num
 
 
conn = psycopg2.connect("host='10.0.0.41' dbname='serverinfo' user='gmastrokostas'")
cur = conn.cursor() #Create the cursor
#Create a Dictionary to pass the value of each function.
server_info = {'hostname': Host_name(), 'IP': IP_address(),'MEM_USE':memory_used_frmt, 'HD_USE':hd_used_frmt, 'CPU_USE':CPU_general_usage(), 'SWP_USE':Swap_memory_usage()}
cur.execute("INSERT INTO SERVER_PERF(hostname, IP, MEM_USE, HD_USE, CPU_USE, SWP_USE) VALUES('%s','%s','%s','%s','%s','%s')" % (Host_name(),IP_address(), memory_used_frmt, hd_used_frmt, CPU_general_usage(), Swap_memory_usage()  ))
 
#If this is not present the changes will not get commited.
conn.commit()

The Install.py Script.

#!/usr/bin/python
import subprocess
import os
import pwd
import grp
from pwd import getpwnam
import psycopg2
import re
import time

def install_PostgresSQL():
#This function checks if the Postgres RPM package is installed or not.
#If not it will then download the RPM package and then installs the packages nde     d
#for the postgresSQL database.
    try:

        if subprocess.call(["rpm", "-q", "pgdg-centos94"]) == 0:
            print "POSTGRES-SQL IS ALREADY INSTALLED"

        elif subprocess.call(["rpm", "-q", "pgdg-centos94"]) == 1:
            subprocess.call(["yum", "localinstall", "-y", "http://yum.postgresql.org/9.4/redhat/rhel-7-x86_64/pgdg-centos94-9.4-1.noarch.rpm"])
            subprocess.call(["yum", "install", "-y", "postgresql94-contrib", "postgresql94-server"])
            print "#################################################################"
            post_install_PostgreSQL_create_postgres_Directories()

    except Exception as e:

        print e
        print "Install of RPM Packages failed. Exiting with Error at function Install_PostgresSQL"
        exit(0)

def post_install_PostgreSQL_create_postgres_Directories():
#This function will create the directories from where the database will run fr.
#It will also adjust the permissions of these directories so they will be owneby      user postgres.
#It will also initialize the database to the directory created and also start e      database.
    try:
        #Setup directories and set permissions
        parent_dir = '/home/postgres'
        directory = '/home/postgres/postgres94'
        postgres_user = pwd.getpwnam('postgres')[2]
        postgres_group = grp.getgrnam('postgres')[2]

        #Check if the postgres home directory exists. If it doesn't create it. If it does then exit.
        if not os.path.exists(directory):
            os.makedirs(directory)
            os.chown(parent_dir, postgres_user, postgres_group)
        else:
            exit(0)
        #Change Recursively all permissions of the /home/postgres directory to be owned by user postgres
        for dirpath, dirname, filenames in os.walk(parent_dir):

            for loop_dir in dirname:
                path = os.path.join(dirpath, loop_dir)
                os.chown(path, postgres_user, postgres_user )

        post_install_PostgreSQL_start_database()

    except Exception as e:

        print e
        print "Error in Post Install steps. Exiting with Error at function post_install_PostgreSQL_create_postgres_Directories "

def post_install_PostgreSQL_start_database():
    try:
        os.system("su postgres -c '/usr/pgsql-9.4/bin/initdb -D /home/postgres/postgres94'")
        #os.system("su postgres -c '/usr/pgsql-9.4/bin/pg_ctl -D /home/postgres/postgres94 -l /home/postgres/postgres94/logfile start'")
        print "INITDB DATABASE"
        post_install_PostgreSQL_configure_postgres_files()

    except  Exception as e:
        print e
        print "Exiting with error at function post_install_PostgreSQL_start_database"

def post_install_PostgreSQL_configure_postgres_files():

    try:
        ##Configure the listening address and the port on the postgres.conf file
        text_ip = "#listen_addresses = 'localhost'"
        subtext_ip = "listen_addresses = '10.0.0.27'"
        text_port = "#port = 5432"
        subtext_port = "port = 5432"

        s = open("/home/postgres/postgres94/postgresql.conf").read()
        s = s.replace(text_ip, subtext_ip)
        s = s.replace(text_port, subtext_port)
        f = open("/home/postgres/postgres94/postgresql.conf", 'w')
        f.write(s)
        f.close()


        s = open("/home/postgres/postgres94/postgresql.conf").read()
        s = s.replace(text_port, subtext_port)
        f = open("/home/postgres/postgres94/postgresql.conf", 'w')
        f.write(s)
        f.close()
        ###########END#########################

        #Edit the pg_hba.conf file to setup from whom the server will accept connections from
        list = ["host    all             all             10.0.0.0/26             trust", "host    all             all             127.0.0.1/32            trust"]

        f = open('/home/postgres/postgres94/pg_hba.conf', "a")
        for items in list:

            f.write(items+'\n')
            f.close
        #########END#########################

        time.sleep(30)

    except Exception as e:
        print e
        print "Exiting with error at function post_install_PostgreSQL_configure_postgres_files"



def create_Database():

    try:
        os.system("su postgres -c '/usr/pgsql-9.4/bin/pg_ctl -m fast -D /home/postgres/postgres94  -l /home/postgres/postgres94/logfile start'")
        print "STARTING DATABASE FOR THE FIRST TIME"
        time.sleep(60)
        conn = psycopg2.connect("host='10.0.0.27' dbname='postgres' user='postgres'")
        cur = conn.cursor() #Create the cursor
        conn.autocommit = True
        cur.execute("""CREATE DATABASE SERVERS""")
        cur.execute("""CREATE ROLE serverseek WITH PASSWORD 'Password!'""")
        cur.execute("""ALTER DATABASE SERVERS OWNER TO SERVERSEEK""")
        cur.execute("""ALTER ROLE SERVERSEEK WITH LOGIN""")
        cur.execute("""GRANT ALL PRIVILEGES ON DATABASE SERVERS to serverseek""")
        conn.commit()
        conn.close()

        conn = psycopg2.connect("host='10.0.0.27' dbname='servers' user='serverseek'")
        cur = conn.cursor() #Create the cursor
        cur.execute("CREATE TABLE SERVERS( HOSTNAME        TEXT NOT NULL, IP              INET NOT NULL, OS              TEXT, OS_VERSION              TEXT, RAM             NUMERIC(10,2), HD_TOTAL_SPACE  NUMERIC(10,2), CPU             TEXT, MAC             MACADDR, TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP, CONSTRAINT PK_HOSTNAME PRIMARY KEY (HOSTNAME));")
        cur.execute("CREATE TABLE SERVER_PERF( HOSTNAME        TEXT NOT NULL REFERENCES SERVERS (HOSTNAME), IP              INET NOT NULL, MEM_USE         NUMERIC(10,2), HD_USE          NUMERIC(10,2), CPU_USE         NUMERIC(10,2), SWP_USE         NUMERIC(10,2), TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP);")
        #If this is not present the changes will not get commited.
        conn.commit()
        conn.close()
        #for row in cur: #Enable the following two lines to view query results
        #    print rows
    except Exception as e:
        print e
        print "Exiting with error at function create_Database"

########################
install_PostgresSQL()
########################
create_Database()

The server_perf.sql script, is the SQL script needed to create the role, database, tables in the postgresSQL database. IMPORTANT If you run the install.py script you do not need to run this SQL script. The install.py does this for you.

#Following SQL script creates role-database and adjusts ownership of the new database.
#As user Postgres Run the following
CREATE DATABASE SERVERS;
CREATE ROLE serverseek WITH PASSWORD 'YourPassword';
ALTER DATABASE SERVERS OWNER TO SERVERSEEK;
ALTER ROLE SERVERSEEK WITH LOGIN;
GRANT ALL PRIVILEGES ON DATABASE SERVERS to SERVERSEEK;

#Following SQL script creates the table needed for the script to insert the data it captures.
#As user  SERVERSEEK run the following.
CREATE TABLE SERVERS
(
HOSTNAME        TEXT NOT NULL,
IP              INET NOT NULL,
OS              TEXT,
OS_VERSION              TEXT,
RAM             NUMERIC(10,2),
HD_TOTAL_SPACE  NUMERIC(10,2),
CPU             TEXT,
MAC             MACADDR,
TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT PK_HOSTNAME PRIMARY KEY (HOSTNAME)
);
ALTER TABLE SERVERS
        OWNER TO SERVERSEEK;

CREATE TABLE SERVER_PERF
(
HOSTNAME        TEXT NOT NULL REFERENCES SERVERS (HOSTNAME),
IP              INET NOT NULL,
MEM_USE         NUMERIC(10,2),
HD_USE          NUMERIC(10,2),
CPU_USE         NUMERIC(10,2),
SWP_USE         NUMERIC(10,2),
TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
ALTER TABLE SERVER_PERF
        OWNER TO SERVERSEEK;



——————————————————————————————————
MODIFIED VERSIONS OF STATIC AND DYNAMIC PYTHON SCRIPTS. MEANT TO BE DEPLOYED ON VERSION 2 OF THE OVERALL PACKAGE
——————————————————————————————————
STATIC.PY


#!/usr/bin/python
import psutil
import os
import math
import platform
import subprocess
import socket
#import psycopg2
import netifaces as ni
import humanize
from cpuinfo import cpuinfo


class Static():
    def __init__(self):
        #NIC INFO
        self.hostname   = socket.gethostname()
        self.iface      = ni.interfaces()[1]
        self.ipaddress  = ni.ifaddresses(self.iface)[2][0]['addr']
        #---OS INFO
        self.osinfo_1   = platform.dist()

        self.osinfo_2_os    = platform.uname()[0]
        self.osinfo_2_ver   = platform.uname()[1]
        self.osinfo_2_rel   = platform.uname()[2]
        #----RAM INFO
        raw_totalM = psutil.virtual_memory().total
        self.total_M    = humanize.naturalsize(raw_totalM)
        #----CPU INFO
        self.info       = cpuinfo.get_cpu_info()
        self.brand      = self.info['brand']
        self.Mhz        = self.info['count']
        self.arch       = self.info['bits']

    def get_OS_make(self):
       if platform.system()     =="Windows":
           return self.osinfo_2_os, self.osinfo_2_ver, self.osinfo_2_rel
       elif platform.system()  =="Linux":
           return self.osinfo_1

host = Static()
'''
print host.hostname
print host.iface
print host.ipaddress
print host.osinfo_1
print host.osinfo_2_os
print host.osinfo_2_ver
print host.osinfo_2_rel
print host.total_M
print host.brand
print host.Mhz
print host.arch
'''
print host.get_OS_make()

DYNAMIC.PY

#THIS FILE IS MANAGED BY PUPPET
#!/usr/bin/python
import psutil
import os
import math
import platform
import subprocess
import socket
#import psycopg2
import netifaces as ni
import humanize
from cpuinfo import cpuinfo

class Dynamic():
    def __init__(self):
        #NIC INFO
        self.hostname   = socket.gethostname()
        self.iface      = ni.interfaces()[1]
        self.ipaddress  = ni.ifaddresses(self.iface)[2][0]['addr']
        #RAM USAGE INFO
        self.total_ram_raw       = humanize.naturalsize(psutil.virtual_memory().total)
        self.used_ram_raw        = humanize.naturalsize(psutil.virtual_memory().used)
        self.used_ram_perc       = psutil.virtual_memory().percent
        #HD USAGE INFO
        self.total_HD_raw        = humanize.naturalsize(psutil.disk_usage('/').total)
        self.used_HD_raw         = humanize.naturalsize(psutil.disk_usage('/').used)
        self.used_HD_perc        = psutil.disk_usage('/').percent
        #CPU USAGE INFO
        self.cpu_use_perc        = psutil.cpu_percent()
        #SWAP USAGE INFO
        self.swap_used           = humanize.naturalsize(psutil.swap_memory().used)
        self.swap_perc           = psutil.swap_memory()[3]
        self.swap_total          = humanize.naturalsize(psutil.swap_memory().total)
    def export_to_csv(self):
        print self.hostname


info = Dynamic()

Share Button

Python – Use threading to get all EOD quote prices from all NASDAQ symbols.

The script below goes to yahoo finance and takes the current price of all stocks from NASDAQ. There is really no real life application of this script. It was created to learn threading. The script obtains the prices for thousands of stocks with in a couple of seconds at most. The output at some sections is off due to the speed the results are being printed. In order for this to work you will have to download all the symbols for the NASDAQ exchange and save that info in a text file located in the same location the script is running from.

#!/usr/bin/python
import urllib
import re
import threading
from datetime import datetime

print "------------------------------------------"
print "Today's date: ", str(datetime.now())[0:10]
print "------------------------------------------"
print ""
print "Symbol    Price      Time"

def stock_prices_reader(smbl):
    url = "http://finance.yahoo.com/q?s=" + smbl + "&ql=0"
    #print url
    html_link = urllib.urlopen(url)
    html_read = html_link.read()
    regex = '(.+?)'
    pattern = re.compile(regex)
    price = re.findall(pattern, html_read)
    print (str(smbl).center(5)), (''.join(price).center(14)), str(datetime.now())[11:25]
    print (" ----".ljust(1)), ("-----".center(14)),("--------".rjust(5))

symbolsfile = open("symbols.txt")
symbolist = symbolsfile.read()
newsymbolist = symbolist.split(("\n"))

threadlist = []
for loop1 in newsymbolist:
    thrd = threading.Thread(target=stock_prices_reader, args=(loop1,))
    thrd.start()
    threadlist.append(thrd)

for loop2 in threadlist:
    loop2.join()

Share Button