2

I am importing a module I created, but none of the print statements seem to be generating output when the main program is run.

Any reason for this?

Main module is as below:

import cx_Oracle
import csv
import os
import glob
import datetime
import multiprocessing as mp
import get_column_stats as gs
import pandas as pd
import pandas.io.sql as psql


def get_data():
    print("Starting Job: " + str(datetime.datetime.now()))

    # Step 1: Init multiprocessing.Pool()   
    pool = mp.Pool(mp.cpu_count())
    print("CPU Count: " + str(mp.cpu_count()))

    dsn_tns = cx_Oracle.makedsn(server.company.net', '1521', service_name='myservice')
    con = cx_Oracle.connect(user='userx', password='mypass', dsn=dsn_tns)


    stats_results = [["OWNER","TABLE","COLUMN_NAME","RECORD_COUNT","DISTINCT_VALUES","MIN_LENGTH","MAX_LENGTH","MIN_VAL","MAX_VAL"]]

    sql = "SELECT * FROM ARIEL.DIM_REGISTRATION_SET"

    cur = con.cursor()
    print("Start Executing SQL: " + str(datetime.datetime.now()))

    df = psql.read_sql(sql, con);

    print("End SQL Execution: " + str(datetime.datetime.now()))

    col_names = df.columns.values.tolist()
    col_index = 0


    print("Start In-Memory Iteration of Dataset: " + str(datetime.datetime.now()))
    # we go through every field

    # start at column 0
    col_index = 0

    # iterate through each column, to gather stats from each column using parallelisation
    proc_results = pool.map_async(gs.get_column_stats, df.iteritems()).get()
    print(type(proc_results))
    print(proc_results)
    print('------------------')
    print(stats_results)
    for result in proc_results:
        stats_results.append(result)
    print('------------------')
    print('------------------')
    print(stats_results)
    print('------------------')
    print('------------------')
    print('------------------')



    # Step 3: Don't forget to close
    pool.close() 
    pool.join()


    print("End In-Memory Iteration of Dataset: " + str(datetime.datetime.now()))
    # end filename for
    cur.close()           

    outfile = open('C:\jupyter\Experiment\stats_dim_registration_set.csv','w')
    writer=csv.writer(outfile,quoting=csv.QUOTE_ALL, lineterminator='\n')
    writer.writerows(stats_results)
    outfile.close()
    print("Ending Job: " + str(datetime.datetime.now()))





get_data()

The imported get_stats module contains this, and is not producing any print statements to the console:

def strip_crlf(value):
    return value.replace('\n', ' ').replace('\r', '')

def get_column_stats(args):
    # args is a tuple, the first value is the column name of the panda series, the second value is the panda data series
    col_name, rs = args

    print("Starting Iteration of Column: " + col_name)
    max_length = 0 
    min_length = 100000  # abitrarily large number!!

    max_value = ""
    min_value = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"  # abitrarily large number!!

    distinct_value_count = 0

    has_values = False # does the column have any non-null values
    has_null_values = False

    row_count = 0

    # create a dictionary into which we can add the individual items present in each row of data
    # a dictionary will not let us add the same value more than once, so we can simply count the 
    # dictionary values at the end
    distinct_values = {}

    row_index = 0



    # go through every row, for the current column being processed to gather the stats
    for row_value in rs.values:
        row_count += 1


        if row_value is None:
            value_length = 0
        else:
            value_length = len(str(row_value))


        if value_length > max_length:
            max_length = value_length

        if value_length < min_length:
            if value_length > 0:
                min_length = value_length

        if row_value is not None:
            if str(row_value) > max_value:
                max_value = str(row_value)
            if str(row_value) < min_value:
                min_value = str(row_value)

        # capture distinct values
        if row_value is None:
            row_value = "Null"
            has_null_values = True
        else:
            has_values = True
            distinct_values[row_value] = 1

        row_index += 1
        # end row for

    distinct_value_count = len(distinct_values)

    if has_values == False:
        distinct_value_count = None
        min_length = None
        max_length = None
        min_value = None
        max_value = None
    elif has_null_values == True and distinct_value_count > 0:
        distinct_value_count -= 1

    if min_length == 0 and max_length > 0 and has_values == True:
        min_length = max_length


    return ["ARIEL","DIM_REGISTRATION_SET", col_name,row_count, distinct_value_count, min_length, max_length, 
                            strip_crlf(str(min_value)), strip_crlf(str(max_value))]

    print("Ending Iteration of Column: " + col_name)  
petezurich
  • 9,280
  • 9
  • 43
  • 57
smackenzie
  • 2,880
  • 7
  • 46
  • 99
  • You seem to be missing a quote mark inside `get_data`. – khelwood Jun 15 '19 at 15:09
  • 1
    From the first look of things, I see you execute the `gs` module in an asynchronous separate thread. Meaning that code is executed in a different place entirely than your main execution. If you wish to see the print statements, you need to redirect the stdout from the pool into your main app. – Richard Nemeth Jun 15 '19 at 15:12
  • How would I do that? Makes sense now you mention it. – smackenzie Jun 15 '19 at 15:15
  • https://stackoverflow.com/questions/1501651/log-output-of-multiprocessing-process – David Zemens Jun 15 '19 at 15:17
  • https://stackoverflow.com/questions/7714868/python-multiprocessing-how-can-i-reliably-redirect-stdout-from-a-child-process/11779039 – David Zemens Jun 15 '19 at 15:17
  • 1
    Possible duplicate of [Redirect stdout to a file in Python?](https://stackoverflow.com/questions/4675728/redirect-stdout-to-a-file-in-python) – David Zemens Jun 15 '19 at 15:18
  • another possible: https://stackoverflow.com/questions/30793624/grabbing-stdout-of-a-function-with-multiprocessing – wwii Jun 15 '19 at 15:42
  • 1
    Thanks all. Is it not possible to share the main process standard output in a shared memory space? – smackenzie Jun 15 '19 at 20:13

0 Answers0