U
SW5 @ s d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl Z zd dl
m Z W n e k
rx dZ Y nX zd dl
m
Z
W n e k
r d dlm
Z
Y nX zd dlmZ W n e k
r dZY nX d dlmZ d dlmZ zd dlmZ W n6 e k
r4 d dlmZ d d lmZ d
d ZY nX zd d lmZ W n e k
r` dZY nX d
ZdZ dZ dZ dZ dZ i Z!i Z"i Z#G dd de$Z%e% Z&e% Z'dd Z(dd Z)dd Z*dd Z+d d Z,d d Z-d!d" Z.e*edud#d$Z/e*e d%d& Z0e*e dvd(d)Z1e*e d*d+ Z2e*e d,d- Z3ej4d d.krbe*e dwd0d1Z5e*e d2d3 Z6n"e*e dxd4d1Z5e*e d5d3 Z6e*e d6d7 Z7e*e dyd8d9Z8e*e d:d; Z9e*e d<d= Z:e*e d>d? Z;e*e dzd@dAZ<e*e dBdC Z=e*e dDdE Z>e*e dFdG Z?e*e dHdI Z@e*e dJdK ZAdLdM ZBe*e dNdO ZCe*e dPdQ ZDe*e dRdS ZEe*e dTdU ZFe*e dVdW ZGe*e dXdY ZHG dZd[ d[eIZJG d\d] d]eJZKej4dd. d^krd_d` ZLndadb ZLe(e G dcdd ddeKZMe(e G dedf dfeKZNe(e G dgdh dheIZOe(e G didj djeIZPe(e G dkdl dleJZQe(e G dmdn dneJZRe(e G dodp dpeIZSedk re*e ejTZTe*e ejUZUe*e ejVZVe(e ejWZWedk r
e)e G dqdr dreZXe)e G dsdt dteZYdS ){ N)Counter)urlparse)
TableFunction)binary_construct) unicode_typeformat_date_time_sqlite)format_date_time)SQLITE_DATETIME_FORMATSc C s
t | tS N)r r
)Z
date_value r /sqlite_udf.pyr ! s r ) _sqlite_udfZ control_flowdatefileZhelpersmathstringc s@ e Zd Z fddZ fddZ fddZ fddZ ZS ) synchronized_dictc s t t| j|| t | _d S r )superr __init__ threadingZLock_lock)selfargskwargs __class__r r
r 8 s zsynchronized_dict.__init__c
s. | j tt| |W 5 Q R S Q R X d S r )r r r __getitem__r keyr r r
r < s z synchronized_dict.__getitem__c
s0 | j tt| ||W 5 Q R S Q R X d S r )r r r __setitem__)r r valuer r r
r @ s z synchronized_dict.__setitem__c
s. | j tt| |W 5 Q R S Q R X d S r )r r r __delitem__r r r r
r" D s z synchronized_dict.__delitem__)__name__
__module__ __qualname__r r r r"
__classcell__r r r r
r 7 s r c s fdd}|S )Nc s( D ] }t |g t | | q| S r )AGGREGATE_COLLECTION
setdefaultappendklassgroupgroupsr r
decoratorN s z aggregate.<locals>.decoratorr r. r/ r r- r
aggregateM s r1 c s fdd}|S )Nc s( D ] }t |g t | | q| S r )TABLE_FUNCTION_COLLECTIONr( r) r* r- r r
r/ V s z!table_function.<locals>.decoratorr r0 r r- r
table_functionU s r3 c s fdd}|S )Nc s( D ] }t |g t | | q| S r )UDF_COLLECTIONr( r) )fnr, r- r r
r/ ^ s zudf.<locals>.decoratorr r0 r r- r
udf] s r6 c G sT t }|D ]D}t| }|D ]2}t|d|j}||kr|| | |d| qq
d S )Nname)setr' getattrr# addZcreate_aggregate)connr. seenr, klassesr+ r7 r r r
register_aggregate_groupsf s
r? c G sF t }|D ]6}t| }|D ]$}|j|kr||j || qq
d S r )r9 r2 r7 r; register)r< r. r= r, r> r+ r r r
register_table_function_groupsp s
rA c G sL t }|D ]<}t| }|D ]*}|j}||kr|| | |d| qq
d S )Nr8 )r9 r4 r# r; Zcreate_function)r< r. r= r, Z functionsZfunctionr7 r r r
register_udf_groupsy s
rB c C s. t | ft t| ft t| ft d S r )r? r' rA r2 rB r4 )r< r r r
register_all s rC c C s | r|S |S r r )ZcondZtruthyZfalseyr r r
if_then_else s rD c C sL | dd} | d}|dkr*| d | S | d}|dkrH| d | S | S )NT +r8 -
)replacefind)Zdate_strZtz_idx1Ztz_idx2r r r
strip_tz s
rL , c C s^ d}g }|D ]>\}}t | |\}} |r |dkr2dp4d}|d|||f q |sTdS ||S )N))i3year)i ' month)i: Zweek)Q day)i hour)< minute) secondrU s z%s %s%sz 0 seconds)divmodr) join)ZnsecondsZgluepartsZaccumoffsetr7 valsuffixr r r
human_delta s r_ c C s0 zt j| }W n tk
r& Y d S X |d S NrU )ospathsplitext
ValueError)filenameresr r r
file_ext s
rg c
C s< z*t | }| W 5 Q R W S Q R X W n Y nX d S r )openread)re Zfhr r r
file_read s
rj c C s t t| |S r )r zlibcompressdataZ compressionr r r
gzip s rq c C s
t | S r )rm
decompressrp r r r
gunzip s rt c C s t t| |S r )rm rn r ro r r r
rq s c C s t | dS )Nzutf-8)rm rr decoders r r r
rt s c C s t | }|r|jS d S r )r Znetloc)ZurlZ parse_resultr r r
hostname s rv c C s2 | } |d k r|t| < nt| t| < }|S r )lowerSTATEget)r Zonr r r
toggle s
rz c G s0 |st | S t|dkr(|d t | < ndS d S )NrU r F)SETTINGSry len)r r r r r
setting s
r} c C s t d S r )r{ clearr r r r
clear_settings s r c C s t d S r )rx r~ r r r r
clear_toggles s r c C s. |d krd| } }n |d kr d}t | ||S Nr rU )randomZ randrange)startendstepr r r
randomrange s
r c C s* zt | |W S tk
r$ Y d S X d S r )r Zgaussrd )ZmeanZsigmar r r
gauss_distribution s r c C s( z t | W S tk
r" Y d S X d S r )r sqrtrd )nr r r
r s r c C sD z
t | W S tk
r> zt| W Y S Y Y d S X Y nX d S r )intrd floatrW r r r
tonumber s
r c C s | r|s dS | |S Nr )count)haystackZneedler r r
substr_count s r c C s t | |S r )r strip)r charsr r r
strip_chars s r c G s" | }|D ]}| | q
| S r )updateZ hexdigest) constructorr Zhash_objargr r r
_hash s r c G s
t tjS r )r hashlibmd5Zvalsr r r
r # s r c G s
t tjS r )r r sha1r r r r
r ' s r c G s
t tjS r )r r sha256r r r r
r + s r c G s
t tjS r )r r sha512r r r r
r / s r c C s
t | S r )rm adler32r r r r
r 3 s r c C s
t | S r )rm crc32r r r r
r 7 s r c @ s$ e Zd Zdd Zdd Zdd ZdS ) _heap_aggc C s g | _ d| _d S r )heapctr r r r
r = s z_heap_agg.__init__c C s |S r r r r! r r r
processA s z_heap_agg.processc C s& | j d7 _ t| j| | d S r` )r heapqheappushr r r r r r
r D s z_heap_agg.stepN)r# r$ r% r r r r r r r
r < s r c @ s e Zd Zdd ZdS )_datetime_heap_aggc C s t |S r r r r r r
r I s z_datetime_heap_agg.processN)r# r$ r% r r r r r
r H s r )rk c C s | j | jd | jd S )NrP g .A)secondsZdaysZ microsecondstdr r r
total_secondsM s
r c C s | S r )r r r r r
<lambda>R r c @ s e Zd Zdd ZdS )mintdiffc C sn d }}| j rZ|d kr,|d kr,t| j }qt| j }|| }|d ksP||krT|}|}q|d k rjt|S d S r )r r heappopr )r dtpmin_diffdtdiffr r r
finalizeV s zmintdiff.finalizeNr# r$ r% r r r r r
r T s r c @ s e Zd Zdd ZdS )avgtdiffc C s | j dk rd S | j dkr dS d }}d }| jrz|dkrL|d krLt| j}q(t| j}|| }|d7 }|t|7 }|}q(t|| S NrU r )r r r r r r )r totalr r r r r r r
r g s"
zavgtdiff.finalizeNr r r r r
r e s r c @ s$ e Zd Zdd Zdd Zdd ZdS )durationc C s d | _ | _d S r _min_maxr r r r
r s zduration.__init__c C s@ t |}| jd ks || jk r"|| _| jd ks6|| jkr<|| _d S r )r r r )r r! r r r r
r s
z
duration.stepc C s$ | j r | jr | j| j }t|S d S r )r r r )r r r r r
r s zduration.finalizeN)r# r$ r% r r r r r r r
r } s r c @ sB e Zd Zer&dd Zdd Zdd Zndd Zdd Zd d Zd
S ) modec C s t | _d S r )r itemsr r r r
r s z
mode.__init__c G s | j | d S r )r r )r r r r r
r s z mode.stepc C s | j r| j dd d S d S r )r most_commonr r r r
r s z
mode.finalizec C s
g | _ d S r )r r r r r
r s c C s | j | d S r )r r) )r itemr r r
r s c C s | j r tt| j | j jdS d S )N)r )r maxr9 r r r r r
r s N)r# r$ r% r r r r r r r r
r s
r c @ s e Zd Zdd ZdS )minrangec C sz | j dkrd S | j dkr dS d }}| jrv|d krH|d krHt| j}q$t| j}|| }|d ksl||krp|}|}q$|S r )r r r r )r prevr currr r r r
r s
zminrange.finalizeNr r r r r
r s r c @ s e Zd Zdd ZdS )avgrangec C s | j dkrd S | j dkr dS d }}d }| jrv|dkrL|d krLt| j}q(t| j}|| }|d7 }||7 }|}q(t|| S r )r r r r r )r r r r r r r r r
r s"
zavgrange.finalizeNr r r r r
r s r c @ s( e Zd ZdZdd Zdd Zdd ZdS ) _rangerangec C s d | _ | _d S r r r r r r
r s z_range.__init__c C s8 | j d ks|| j k r|| _ | jd ks.|| jkr4|| _d S r r r r r r
r s z _range.stepc C s$ | j d k r | jd k r | j| j S d S r r r r r r
r s z_range.finalizeN)r# r$ r% r7 r r r r r r r
r s r c @ s0 e Zd ZddgZdgZdZd
ddZdd ZdS ) RegexSearchregex
search_stringmatchZ regex_searchNc C s t ||| _d S r )refinditer_iter)r r r r r r
initialize s zRegexSearch.initializec C s t | jdfS r )nextr r, )r idxr r r
iterate s zRegexSearch.iterate)NNr# r$ r% Zparamscolumnsr7 r r r r r r
r s
r c @ s2 e Zd ZdddgZdgZdZd ddZd d
Zd S )
DateSeriesr stop step_secondsr Z date_seriesrP c C s t || _t || _t|}tj|d| _| jjdkr^| jjdkr^| jj dkr^|dkr^d| _
n^| jj dkr| jj dkr| jj
dkr| jj dkr| jj dkr| jj
dkr|dk rd| _
nd| _
d S ) N)r r rP z%Y-%m-%dil rU z%H:%M:%Sz%Y-%m-%d %H:%M:%S)r r r r datetimeZ timedeltar rR rT rV formatrN rO rQ )r r r r r r r
r s6
zDateSeries.initializec C s4 | j | jkrt| j }| j | j7 _ || jfS r )r r
StopIterationr strftimer )r r Zcurrentr r r
r s
zDateSeries.iterateN)rP r r r r r
r s
r )N)rM )rl )rl )N)NN)Zr r r r ra r r sysr rm collectionsr ImportErrorr Z urllib.parseZvtfuncr Zpeeweer r Zplayhouse._speedupsr r r
Z playhouser Z
cython_udfZ CONTROL_FLOWZDATEFILEZHELPERZMATHSTRINGr' r2 r4 dictr rx r{ r1 r3 r6 r? rA rB rC rD rL r_ rg rj version_inforq rt rv rz r} r r r r r r r r r r r r r r r objectr r r r r r r r r r Zdamerau_levenshtein_distZlevenshtein_distZstr_distZmedianr r r r r r
<module> s