Python – Get list of processes, their owners and RAM usage from a Linux system.

This script only works on Linux systems. It lists the running PIDs, the owners of the PIDs and the RAM usage of each PID.

import os, sys
from os.path import join,getsize
import humanize
import pwd
import psutil

pids = [int(pid) for pid in os.listdir('/proc') if pid.isdigit()]
for elements in pids:
    p = psutil.Process(elements)
    proc_name  = p.name()
    proc_stat_file = os.stat("/proc/%d" % elements)
    uid = proc_stat_file.st_uid
    username = pwd.getpwuid(uid)[0]
    human_size = humanize.naturalsize(p.memory_info_ex()[0], gnu=True)
    #print  p.memory_info_ex()[0]
    #print username,"\t\t", elements,"\t\t", p.name(), "\t\t", human_size
    print username.ljust(20), elements, p.name(), human_size.rjust(20)

Output Example

root                 17601 cupsd                 3.4M
gmastrokostas        23460 chrome               117.5M
gmastrokostas        24034 pithos                76.2M
Share Button

Python – Get a listing of all subdirectories and their size on a Linux system

The script asks for a directory from the user. It then will dive in all the subdirectories within that directory. It will list all subdirectories and their size. It does this by actually looking for files within the subdirectories and calculating the actual size (not block size) of all the files with in that subdirectory.

#!/usr/bin/python
import humanize
import os, sys
from os.path import join,getsize
import humanize
import pwd

def dir_list():
    list = []
    drct = raw_input(":Enter directory name. Use full path: ")
    for dirpath, dirnames, filenames in os.walk(drct, followlinks=False):
        for loop_dir in dirnames:
            path = os.path.join(dirpath, loop_dir) #Joins the names of directories with the actual path
            list.append(path) # Enters all the directories with the full path to a list

    return list  #Return the whole list so dir_size function can process it

def dir_size():
    
    returned_list = dir_list() #The list returned from the dir_list function

    for loop in returned_list: #Breaks down the list in strings in order for OS.WALK to be able to process it
        total = 0
        for dirpath, dirnames, filenames in os.walk(loop, followlinks= False): #grab the paths from the list. This is the same as asking a user to enter a path

            for f_name in filenames: #Dive into directories to view files
                path = os.path.join(dirpath, f_name) #Join path with filenames for the directory we are trying to find the size
                if os.path.islink(path): 
                    continue
                else:
                    total += os.path.getsize(path)  #Find size of actual files within the directory
                    human_size = humanize.naturalsize(total, gnu=True) 
        print "Size of directory  : ",loop, "is", human_size
        human_size = 0

dir_size()

OutPut Example

FUN 1:Enter directory name. Use full path: /home/gmastrokostas
Size of directory  :  /home/gmastrokostas/.mozilla is 16.1M
Size of directory  :  /home/gmastrokostas/.config is 69.9K
Size of directory  :  /home/gmastrokostas/.cache is 53.0M
Size of directory  :  /home/gmastrokostas/Desktop is 0
Size of directory  :  /home/gmastrokostas/Downloads is 332.9M
Size of directory  :  /home/gmastrokostas/Templates is 0
Size of directory  :  /home/gmastrokostas/Public is 0
Size of directory  :  /home/gmastrokostas/Documents is 218.6M
Size of directory  :  /home/gmastrokostas/Music is 0
Size of directory  :  /home/gmastrokostas/Pictures is 0
Size of directory  :  /home/gmastrokostas/Videos is 0
Size of directory  :  /home/gmastrokostas/.local is 1.2M
Share Button

Python /PostgresqSQL 9.4 – Server Performance Data Capture V.1

These scripts gather static and dynamic information from servers and insert that data into a PostgresSQL database. The static information is information that unless a major upgrade takes place it never changes. They Dynamic data is performance data of the servers. The purpose of the static data is to be able to query for dynamic information which being inserted to the database every X amount of minutes via cron.

These scripts work only on Linux/Unix based machines.

The static information for the remote servers are as follows :

  • Hostname
  • IP
  • Operating System
  • Operating System Version
  • Total Amount of RAM
  • Total Amount of HD space
  • CPU Type
  • MAC ADDRESS

The dynamic information for the remote servers are as follows :

  • Memory Usage
  • Storage Usage
  • Swap Usage
  • Time the data was captured

The static.py script will need to be run only one time on the remote servers or when a major upgrade occurs that might change configuration regarding RAM, Partitions, IP, Operating System (even an upgrade), CPU, NIC replacement.

The dynamic.py script will be run on the remote servers via cron. It is the script that captures information which is being constantly changed, like memory, storage, swap usage. All this data is sent for insertion to the remote database. The script executes it self via cron and then sent to be inserted into a PostgreSQL database.

In both scripts data is entered into a dictionary and then a connection to the database is created in order to insert the data.

The install.py script is not required to be run but it does several things:

  • Installs the PostgresSQL 9.4 database.
  • Creates the database and the role assigned to the database.
  • Creates the Tables needed
  • Configures the postgres.conf file to listen to a specific IP address and port.

However, if you wish to manually setup the database, the SQL script is provided at the bottom of this page. The script is called server_perf.sql

The static.py Script.

#!/usr/bin/python
import psutil
import os
import math
import platform
import subprocess
import socket
import psycopg2
import netifaces as ni

from decimal import *

#Used by the convert values function. It is needed to provide human readable data for the RAM amount produced.
factor = 1024

def Host_name():
    #Get the hostname
    hostname = socket.gethostname()
    return hostname

def IP_address():
    #Get the IP for ETH0
    ni.ifaddresses('eth0')
    ip = ni.ifaddresses('eth0')[2][0]['addr']
    return ip

def OS_make():
    #CPU INFORMATION. Provides the OS make (Windows, Ubunty, RH, Centos, etc)
    #cpu_version = platform.dist()[0]+" "+platform.dist()[1]
    cpu_version = platform.dist()[0]
    return  cpu_version

def OS_Version():
    #Provides the version of the OS.
    os_version = platform.dist()[1]
    return os_version

def Virtual_memory_total():
    #Provides total amount of RAM
    cvr_info = psutil.virtual_memory().total
    return round(cvr_info, 2)

def convert_values():
    #Converts the KB to GB and presents it as such.
    cvr_info = Virtual_memory_total()
    i = int(math.log(cvr_info)/math.log(factor))
    result = float(cvr_info)/factor**i
    return result

def get_processor_info():
    #Gets the verbose info about the CPU (AMD FX(tm)-6350)
    if platform.system() == "Windows":
        return platform.processor()
    elif platform.system() == "Darwin":
        return subprocess.check_output(['/usr/sbin/sysctl', "-n", "machdep.cpu.brand_string"]).strip()
    elif platform.system() == "Linux":
        command = "cat /proc/cpuinfo | grep 'model name' | head -1 | awk -F: '{print $2}'"
        return subprocess.check_output(command, shell=True).strip()
    return ""

#Adjusting precision and then appending to list information taken from Virtual_memory_total function
mem_frmt =   "{0:.2f}".format(convert_values())

#Create the Database PostgreSQL 9.4 connection.
conn = psycopg2.connect("host='10.0.0.41' dbname='serverinfo' user='gmastrokostas'")
cur = conn.cursor() #Create the cursor
#Create a Dictionary to pass the value of each function.
server_info = {'hostname': Host_name(), 'IP': IP_address(), 'OS':OS_make(), 'OS_V':OS_Version(), 'RAM':mem_frmt, 'CPU':get_processor_info()}
cur.execute("INSERT INTO servers(hostname, IP, OS, OS_Version, RAM, CPU) VALUES('%s','%s','%s','%s','%s','%s')" % (Host_name(),IP_address(), OS_make(), OS_Version(), mem_frmt, get_processor_info() ))

#If this is not present the changes will not get commited.
conn.commit()

The Dynamic.py script

import psutil
import os
import math
import platform
import subprocess
import socket
import psycopg2
import netifaces as ni
 
def Host_name():
    #Get the hostname
    hostname = socket.gethostname()
    return hostname
 
def IP_address():
    #Get the IP for ETH0
    ni.ifaddresses('eth0')
    ip = ni.ifaddresses('eth0')[2][0]['addr']
    return ip
 
factor = 1024
def Virtual_memory_use():
    #Provides total amount of RAM
    cvr_info = psutil.virtual_memory().used
    #cvr_info_ram = Virtual_memory_use()
    i = int(math.log(cvr_info)/math.log(factor))
    result_ram = float(cvr_info)/factor**i
    return round(result_ram)
memory_used_frmt =   "{0:.2f}".format(Virtual_memory_use())
 
def Partition_root_Usedspace():
    #Provides total amount of RAM
    cvr_info = psutil.disk_usage('/').used
    #cvr_info_ram = Virtual_memory_use()
    i = int(math.log(cvr_info)/math.log(factor))
    result_ram = float(cvr_info)/factor**i
    return round(result_ram)
hd_used_frmt =   "{0:.2f}".format(Partition_root_Usedspace())
 
 
def CPU_general_usage():
    #CPU INFORMATION.
    cpu_prec_usage = psutil.cpu_percent(interval=0)
    return cpu_prec_usage
 
def Swap_memory_usage():
    #SWAP SIZE USAGE
    swap_mem_full_num = psutil.swap_memory().used
    return swap_mem_full_num
 
 
conn = psycopg2.connect("host='10.0.0.41' dbname='serverinfo' user='gmastrokostas'")
cur = conn.cursor() #Create the cursor
#Create a Dictionary to pass the value of each function.
server_info = {'hostname': Host_name(), 'IP': IP_address(),'MEM_USE':memory_used_frmt, 'HD_USE':hd_used_frmt, 'CPU_USE':CPU_general_usage(), 'SWP_USE':Swap_memory_usage()}
cur.execute("INSERT INTO SERVER_PERF(hostname, IP, MEM_USE, HD_USE, CPU_USE, SWP_USE) VALUES('%s','%s','%s','%s','%s','%s')" % (Host_name(),IP_address(), memory_used_frmt, hd_used_frmt, CPU_general_usage(), Swap_memory_usage()  ))
 
#If this is not present the changes will not get commited.
conn.commit()

The Install.py Script.

#!/usr/bin/python
import subprocess
import os
import pwd
import grp
from pwd import getpwnam
import psycopg2
import re
import time

def install_PostgresSQL():
#This function checks if the Postgres RPM package is installed or not.
#If not it will then download the RPM package and then installs the packages nde     d
#for the postgresSQL database.
    try:

        if subprocess.call(["rpm", "-q", "pgdg-centos94"]) == 0:
            print "POSTGRES-SQL IS ALREADY INSTALLED"

        elif subprocess.call(["rpm", "-q", "pgdg-centos94"]) == 1:
            subprocess.call(["yum", "localinstall", "-y", "http://yum.postgresql.org/9.4/redhat/rhel-7-x86_64/pgdg-centos94-9.4-1.noarch.rpm"])
            subprocess.call(["yum", "install", "-y", "postgresql94-contrib", "postgresql94-server"])
            print "#################################################################"
            post_install_PostgreSQL_create_postgres_Directories()

    except Exception as e:

        print e
        print "Install of RPM Packages failed. Exiting with Error at function Install_PostgresSQL"
        exit(0)

def post_install_PostgreSQL_create_postgres_Directories():
#This function will create the directories from where the database will run fr.
#It will also adjust the permissions of these directories so they will be owneby      user postgres.
#It will also initialize the database to the directory created and also start e      database.
    try:
        #Setup directories and set permissions
        parent_dir = '/home/postgres'
        directory = '/home/postgres/postgres94'
        postgres_user = pwd.getpwnam('postgres')[2]
        postgres_group = grp.getgrnam('postgres')[2]

        #Check if the postgres home directory exists. If it doesn't create it. If it does then exit.
        if not os.path.exists(directory):
            os.makedirs(directory)
            os.chown(parent_dir, postgres_user, postgres_group)
        else:
            exit(0)
        #Change Recursively all permissions of the /home/postgres directory to be owned by user postgres
        for dirpath, dirname, filenames in os.walk(parent_dir):

            for loop_dir in dirname:
                path = os.path.join(dirpath, loop_dir)
                os.chown(path, postgres_user, postgres_user )

        post_install_PostgreSQL_start_database()

    except Exception as e:

        print e
        print "Error in Post Install steps. Exiting with Error at function post_install_PostgreSQL_create_postgres_Directories "

def post_install_PostgreSQL_start_database():
    try:
        os.system("su postgres -c '/usr/pgsql-9.4/bin/initdb -D /home/postgres/postgres94'")
        #os.system("su postgres -c '/usr/pgsql-9.4/bin/pg_ctl -D /home/postgres/postgres94 -l /home/postgres/postgres94/logfile start'")
        print "INITDB DATABASE"
        post_install_PostgreSQL_configure_postgres_files()

    except  Exception as e:
        print e
        print "Exiting with error at function post_install_PostgreSQL_start_database"

def post_install_PostgreSQL_configure_postgres_files():

    try:
        ##Configure the listening address and the port on the postgres.conf file
        text_ip = "#listen_addresses = 'localhost'"
        subtext_ip = "listen_addresses = '10.0.0.27'"
        text_port = "#port = 5432"
        subtext_port = "port = 5432"

        s = open("/home/postgres/postgres94/postgresql.conf").read()
        s = s.replace(text_ip, subtext_ip)
        s = s.replace(text_port, subtext_port)
        f = open("/home/postgres/postgres94/postgresql.conf", 'w')
        f.write(s)
        f.close()


        s = open("/home/postgres/postgres94/postgresql.conf").read()
        s = s.replace(text_port, subtext_port)
        f = open("/home/postgres/postgres94/postgresql.conf", 'w')
        f.write(s)
        f.close()
        ###########END#########################

        #Edit the pg_hba.conf file to setup from whom the server will accept connections from
        list = ["host    all             all             10.0.0.0/26             trust", "host    all             all             127.0.0.1/32            trust"]

        f = open('/home/postgres/postgres94/pg_hba.conf', "a")
        for items in list:

            f.write(items+'\n')
            f.close
        #########END#########################

        time.sleep(30)

    except Exception as e:
        print e
        print "Exiting with error at function post_install_PostgreSQL_configure_postgres_files"



def create_Database():

    try:
        os.system("su postgres -c '/usr/pgsql-9.4/bin/pg_ctl -m fast -D /home/postgres/postgres94  -l /home/postgres/postgres94/logfile start'")
        print "STARTING DATABASE FOR THE FIRST TIME"
        time.sleep(60)
        conn = psycopg2.connect("host='10.0.0.27' dbname='postgres' user='postgres'")
        cur = conn.cursor() #Create the cursor
        conn.autocommit = True
        cur.execute("""CREATE DATABASE SERVERS""")
        cur.execute("""CREATE ROLE serverseek WITH PASSWORD 'Password!'""")
        cur.execute("""ALTER DATABASE SERVERS OWNER TO SERVERSEEK""")
        cur.execute("""ALTER ROLE SERVERSEEK WITH LOGIN""")
        cur.execute("""GRANT ALL PRIVILEGES ON DATABASE SERVERS to serverseek""")
        conn.commit()
        conn.close()

        conn = psycopg2.connect("host='10.0.0.27' dbname='servers' user='serverseek'")
        cur = conn.cursor() #Create the cursor
        cur.execute("CREATE TABLE SERVERS( HOSTNAME        TEXT NOT NULL, IP              INET NOT NULL, OS              TEXT, OS_VERSION              TEXT, RAM             NUMERIC(10,2), HD_TOTAL_SPACE  NUMERIC(10,2), CPU             TEXT, MAC             MACADDR, TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP, CONSTRAINT PK_HOSTNAME PRIMARY KEY (HOSTNAME));")
        cur.execute("CREATE TABLE SERVER_PERF( HOSTNAME        TEXT NOT NULL REFERENCES SERVERS (HOSTNAME), IP              INET NOT NULL, MEM_USE         NUMERIC(10,2), HD_USE          NUMERIC(10,2), CPU_USE         NUMERIC(10,2), SWP_USE         NUMERIC(10,2), TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP);")
        #If this is not present the changes will not get commited.
        conn.commit()
        conn.close()
        #for row in cur: #Enable the following two lines to view query results
        #    print rows
    except Exception as e:
        print e
        print "Exiting with error at function create_Database"

########################
install_PostgresSQL()
########################
create_Database()

The server_perf.sql script, is the SQL script needed to create the role, database, tables in the postgresSQL database. IMPORTANT If you run the install.py script you do not need to run this SQL script. The install.py does this for you.

#Following SQL script creates role-database and adjusts ownership of the new database.
#As user Postgres Run the following
CREATE DATABASE SERVERS;
CREATE ROLE serverseek WITH PASSWORD 'YourPassword';
ALTER DATABASE SERVERS OWNER TO SERVERSEEK;
ALTER ROLE SERVERSEEK WITH LOGIN;
GRANT ALL PRIVILEGES ON DATABASE SERVERS to SERVERSEEK;

#Following SQL script creates the table needed for the script to insert the data it captures.
#As user  SERVERSEEK run the following.
CREATE TABLE SERVERS
(
HOSTNAME        TEXT NOT NULL,
IP              INET NOT NULL,
OS              TEXT,
OS_VERSION              TEXT,
RAM             NUMERIC(10,2),
HD_TOTAL_SPACE  NUMERIC(10,2),
CPU             TEXT,
MAC             MACADDR,
TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT PK_HOSTNAME PRIMARY KEY (HOSTNAME)
);
ALTER TABLE SERVERS
        OWNER TO SERVERSEEK;

CREATE TABLE SERVER_PERF
(
HOSTNAME        TEXT NOT NULL REFERENCES SERVERS (HOSTNAME),
IP              INET NOT NULL,
MEM_USE         NUMERIC(10,2),
HD_USE          NUMERIC(10,2),
CPU_USE         NUMERIC(10,2),
SWP_USE         NUMERIC(10,2),
TIME_CAPTURED   TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
ALTER TABLE SERVER_PERF
        OWNER TO SERVERSEEK;



——————————————————————————————————
MODIFIED VERSIONS OF STATIC AND DYNAMIC PYTHON SCRIPTS. MEANT TO BE DEPLOYED ON VERSION 2 OF THE OVERALL PACKAGE
——————————————————————————————————
STATIC.PY


#!/usr/bin/python
import psutil
import os
import math
import platform
import subprocess
import socket
#import psycopg2
import netifaces as ni
import humanize
from cpuinfo import cpuinfo


class Static():
    def __init__(self):
        #NIC INFO
        self.hostname   = socket.gethostname()
        self.iface      = ni.interfaces()[1]
        self.ipaddress  = ni.ifaddresses(self.iface)[2][0]['addr']
        #---OS INFO
        self.osinfo_1   = platform.dist()

        self.osinfo_2_os    = platform.uname()[0]
        self.osinfo_2_ver   = platform.uname()[1]
        self.osinfo_2_rel   = platform.uname()[2]
        #----RAM INFO
        raw_totalM = psutil.virtual_memory().total
        self.total_M    = humanize.naturalsize(raw_totalM)
        #----CPU INFO
        self.info       = cpuinfo.get_cpu_info()
        self.brand      = self.info['brand']
        self.Mhz        = self.info['count']
        self.arch       = self.info['bits']

    def get_OS_make(self):
       if platform.system()     =="Windows":
           return self.osinfo_2_os, self.osinfo_2_ver, self.osinfo_2_rel
       elif platform.system()  =="Linux":
           return self.osinfo_1

host = Static()
'''
print host.hostname
print host.iface
print host.ipaddress
print host.osinfo_1
print host.osinfo_2_os
print host.osinfo_2_ver
print host.osinfo_2_rel
print host.total_M
print host.brand
print host.Mhz
print host.arch
'''
print host.get_OS_make()

DYNAMIC.PY

#THIS FILE IS MANAGED BY PUPPET
#!/usr/bin/python
import psutil
import os
import math
import platform
import subprocess
import socket
#import psycopg2
import netifaces as ni
import humanize
from cpuinfo import cpuinfo

class Dynamic():
    def __init__(self):
        #NIC INFO
        self.hostname   = socket.gethostname()
        self.iface      = ni.interfaces()[1]
        self.ipaddress  = ni.ifaddresses(self.iface)[2][0]['addr']
        #RAM USAGE INFO
        self.total_ram_raw       = humanize.naturalsize(psutil.virtual_memory().total)
        self.used_ram_raw        = humanize.naturalsize(psutil.virtual_memory().used)
        self.used_ram_perc       = psutil.virtual_memory().percent
        #HD USAGE INFO
        self.total_HD_raw        = humanize.naturalsize(psutil.disk_usage('/').total)
        self.used_HD_raw         = humanize.naturalsize(psutil.disk_usage('/').used)
        self.used_HD_perc        = psutil.disk_usage('/').percent
        #CPU USAGE INFO
        self.cpu_use_perc        = psutil.cpu_percent()
        #SWAP USAGE INFO
        self.swap_used           = humanize.naturalsize(psutil.swap_memory().used)
        self.swap_perc           = psutil.swap_memory()[3]
        self.swap_total          = humanize.naturalsize(psutil.swap_memory().total)
    def export_to_csv(self):
        print self.hostname


info = Dynamic()

Share Button