ValueError: Error parsing datetime string NumPy

Question

I am trying to convert this string date into a date format in a NumPy array. I am using the datetime64 datatype cast off to seconds and receive this error. My code is below. I would like to write the numpy datatype as a date type to my database.

import json
import jsonpickle
import requests
import arcpy
import numpy as np    #NOTE THIS
import random
import timestring




fc = "C:\MYLATesting.gdb\MYLA311Copy"
if arcpy.Exists(fc):
  arcpy.Delete_management(fc)





f2 = open('C:\Users\Administrator\Desktop\DetailView.json', 'r')
data2 = jsonpickle.encode( jsonpickle.decode(f2.read()) )

url2 = "myURL"
headers2 = {'Content-type': 'text/plain', 'Accept': '/'}

r2 = requests.post(url2, data=data2, headers=headers2)
decoded2 = json.loads(r2.text)


dt = np.dtype([('SRAddress', 'U40'),
                ('LatitudeShape', '<f8'),
                ('LongitudeShape', '<f8'),
                ('Latitude', '<f8'),
                ('Longitude', '<f8'),
                ('Type', 'U40'),
                ('SRNumber', 'U40'),
                ('FirstName', 'U40'),
               ('LastName', 'U40'),
               ('HomePhone', 'U40'),
                ('CreateDate', 'datetime64[S]'),
               ('Comment', 'U128'),
                ('ItemInfo', 'U128'),
                ('DayTest', 'U128'),
                ('DistrictName', 'U128'),
                ('ShortDay', 'U128'),
                ('ParentNumber', 'U128'),
                ('A_Call_No','U128'),
                ('Area', 'U128'),
                ('DirectionSuffix','U128'),
                ('DistrictAbbr', 'U128'),
                ('DistrictNumber', 'U128'),
                ('DistrictOffice', 'U128'),
                ('Fraction', 'U128'),
                ('R_Call_No', 'U128'),
                ('SectionId', 'U128'),
                ('StreetTo', 'U128'),
                ('StreetFrom', 'U128'),
                ('StreetLightId', 'U128'),
                ('StreetLightStatus', 'U128'),
                ('Y_Call_No', 'U128'),
                ('CommunityPlanningArea', 'U128'),
                ('LastUpdatedBy', 'U128'),
                ('BOSRadioHolderName', 'U128'),
                ])







items = []
for sr in decoded2['Response']['ListOfServiceRequest']['ServiceRequest']:
    SRAddress = sr['SRAddress']
    Latitude = sr['Latitude']
    Longitude = sr['Longitude']
    SRNumber = sr['SRNumber']
    FirstName = sr['FirstName']
    LastName = sr['LastName']
    HomePhone = sr['HomePhone']
    CreatedDate = sr['CreatedDate']


    print CreatedDate


    ItemInfo = " "

    for ew in sr["ListOfLa311ElectronicWaste"][u"La311ElectronicWaste"]:
            CommodityType = ew['Type']
            ItemType = ew['ElectronicWestType']
            ItemCount = ew['ItemCount']
            ItemInfo += '{0},  {1}, '.format(ItemType, ItemCount)
            ParentNumber = ew['Name']




    for GIS in sr["ListOfLa311GisLayer"][u"La311GisLayer"]:
            Day = GIS['Day']
            DistrictName = GIS['DistrictName']
            ShortDay = GIS['ShortDay']
            A_Call_No = GIS['A_Call_No']
            Area = GIS['Area']
            DirectionSuffix = GIS['DirectionSuffix']
            DistrictAbbr = GIS['DistrictAbbr']
            DistrictNumber = GIS['DistrictNumber']
            DistrictOffice = GIS['DistrictOffice']
            Fraction = GIS['Fraction']
            R_Call_No = GIS['R_Call_No']
            SectionId = GIS['SectionId']
            StreetFrom = GIS ['StreetFrom']
            StreetTo = GIS ['StreetTo']
            StreetLightId = GIS ['StreetLightId']
            StreetLightStatus = GIS['StreetLightStatus']
            Y_Call_No = GIS ['Y_Call_No']
            CommunityPlanningArea = GIS['CommunityPlanningArea']
            LastUpdatedBy = GIS['LastUpdatedBy']
            BOSRadioHolderName = GIS['BOSRadioHolderName']




    comments =  [ cl['Comment'] for cl in sr["ListOfLa311ServiceRequestNotes"][u"La311ServiceRequestNotes"]]
    print comments
    Comment = ' '.join(comments)


items.append((SRAddress,
                          Latitude,
                         Longitude,
                          Latitude,
                          Longitude,
                          CommodityType,
                          SRNumber,
                         FirstName,
                          LastName,
                          HomePhone,
                          CreatedDate,
                          Comment,
                          ItemInfo,
                          Day,
                          DistrictName,
                          ShortDay,
                          ParentNumber,
                         A_Call_No,
                        Area,
                        DirectionSuffix,
                        DistrictAbbr,
                        DistrictNumber,
                        DistrictOffice,
                        Fraction,
                        R_Call_No,
                        SectionId,
                        StreetFrom,
                        StreetTo,
                        StreetLightId,
                        StreetLightStatus,
                        Y_Call_No,
                        CommunityPlanningArea,
                        LastUpdatedBy,
                        BOSRadioHolderName
))


arr = np.array(items,dtype=dt)
sr = arcpy.SpatialReference(4326)


arcpy.da.NumPyArrayToFeatureClass(arr, fc, ['longitudeshape', 'latitudeshape'], sr )


print json.dumps(decoded2, sort_keys=True, indent=4)

  File "C:/Users/Administrator/Desktop/DevSummitJSON_PySeminar.py", line 166, in <module>
    arr = np.array(items,dtype=dt)
ValueError: Error parsing datetime string "02/17/2015 16:53:25" at position 2

Answer 1

np.datetime64 works with format yyyy-mm-dd hh:mm:ss

use to_datetime() method in pandas as it is more flexible:

import pandas as pd
pd.to_datetime("02/17/2015 16:53:25")

OR

if you still want to use np.datetime64 then:

change the format of date to yyyy-mm-dd hh:mm:ss

for example:

numpy.datetime64("02/17/2015 16:53:25")
#change above format to:
numpy.datetime64("2015-02-17 16:53:25")

Answer 2

Just looking at the error message you get, you've input your datetime in the wrong string format

numpy.datetime64("02/17/2015 16:53:25")
Traceback (most recent call last):
  File "<interactive input>", line 1, in <module>
ValueError: Error parsing datetime string "02/17/2015 16:53:25" at position 2

numpy.datetime64("2015-02-17T16:53:25")
>>> numpy.datetime64('2015-02-17T16:53:25+0100')

Beware that numpy assumes that the time is given in your local timezone (here UTC+1). Append "Z" and it will be interpreted as UTC.

So you have to either change the format of your string, or you could try the solution using pandas presented here which seems is more flexible in interpreting string formats