hnykda

!SESSION 2014-12-13 12:58:37.593 -----------------------------------------------
eclipse.buildId=4.4.1.M20140925-0400
java.version=1.8.0_25
java.vendor=Oracle Corporation
BootLoader constants: OS=linux, ARCH=x86_64, WS=gtk, NL=en_US
Framework arguments:  -product org.eclipse.epp.package.standard.product
Command-line arguments:  -os linux -ws gtk -arch x86_64 -product org.eclipse.epp.package.standard.product

!ENTRY org.eclipse.equinox.p2.publisher.eclipse 4 0 2014-12-13 12:58:40.906
!MESSAGE Unable to acquire PluginConverter service during generation for: /usr/share/eclipse/plugins/colorer.

hnykda

def _load_data(data, n_prev = INIC):
    docX = []
    docY = []
    for i in range(len(data)-n_prev):
        docX.append(data.iloc[i:i+n_prev].as_matrix())
        docY.append(data.iloc[i+n_prev].as_matrix())
    alsX = np.array(docX)
    alsY = np.array(docY)
    
    return(alsX, alsY)

hnykda

----------------------------    LOGCAT    ----------------------------


--------- beginning of system
I/SystemServer(  765): NetworkTimeUpdateService
I/SystemServer(  765): CommonTimeManagementService
I/SystemServer(  765): CertBlacklister
I/SystemServiceManager(  765): Starting com.android.server.dreams.DreamManagerService
I/SystemServer(  765): Assets Atlas Service
I/SystemServiceManager(  765): Starting com.android.server.print.PrintManagerService

hnykda

----------------------------    LOGCAT    ----------------------------


--------- beginning of system
I/auditd  (    0): Starting up
I/audit_log(    0): Previous audit logfile detected, rotating
E/audit_rules(    0): Could not read audit rules /data/misc/audit/audit.rules: No such file or directory
I/irsc_util(    0): irsc tool created:0xb7048020
I/irsc_util(    0): Starting irsc tool
I/irsc_util(    0): Trying to open sec config file

hnykda

jan 07 07:30:39 rpielec systemd[1]: time has been changed
jan 07 07:30:42 rpielec kernel: usb 1-1.3: new high-speed usb device number 6 using dwc_otg
jan 07 07:30:42 rpielec kernel: bluetooth: core ver 2.21
jan 07 07:30:42 rpielec kernel: net: registered protocol family 31
jan 07 07:30:42 rpielec kernel: bluetooth: hci device and connection manager initialized
jan 07 07:30:42 rpielec kernel: bluetooth: hci socket layer initialized
jan 07 07:30:42 rpielec kernel: bluetooth: l2cap socket layer initialized
jan 07 07:30:42 rpielec kernel: bluetooth: sco socket layer initialized
jan 07 07:30:42 rpielec kernel: bluetooth: hci uart driver ver 2.3
jan 07 07:30:42 rpielec kernel: bluetooth: hci uart protocol h4 registered

hnykda

import os
from time import time
import pandas as pd
from memory_profiler import memory_usage

FILENAME='compressed_df'

def get_size(flnm):
    return round(os.path.getsize(flnm) / (1024*1024), 2)

hnykda

$ spss2py -v 0 execute data/q2-scripts-reduced/reduced/1.hdf data/q2-scripts-reduced/processed/2_statements.csv data/q2-scripts-reduced/processed/2_out.hdf
20170703 21:03:05:root:INFO - ######### STARTING #########
20170703 21:03:05:root:DEBUG - Received command line arguments: 
Namespace(data='data/q2-scripts-reduced/reduced/1.hdf', func=<function execute at 0x7f30159812f0>, output='data/q2-scripts-reduced/processed/2_out.hdf', statements=<_io.TextIOWrapper name='data/q2-scripts-reduced/processed/2_statements.csv' mode='r' encoding='UTF-8'>, verbosity=0)
<class 'pandas.core.frame.DataFrame'>
Int64Index: 72423 entries, 0 to 72422
Columns: 4861 entries, A1 to uid
dtypes: float16(4651), float32(7), int16(1), int8(184), object(18)
memory usage: 667.7+ MB
20170703 21:03:07:spss2py.executer:DEBUG - 

hnykda

tribeca_1        | n:    --minUptime not set. Defaulting to: 1000ms
tribeca_1        | warn:    --spinSleepTime not set. Your script will exit if it does not stay up for at least 1000ms
tribeca_1        | {"name":"tribeca:config","hostname":"ec40165fa73a","pid":16,"level":30,"msg":"TradedPair = BTC/USD","time":"2017-08-19T16:23:21.732Z","v":0}
tribeca_1        | {"name":"tribeca:config","hostname":"ec40165fa73a","pid":16,"level":30,"msg":"WebClientUsername = dan","time":"2017-08-19T16:23:21.746Z","v":0}
tribeca_1        | {"name":"tribeca:config","hostname":"ec40165fa73a","pid":16,"level":30,"msg":"WebClientPassword = seznam12","time":"2017-08-19T16:23:21.748Z","v":0}
tribeca_1        | {"name":"tribeca:main","hostname":"ec40165fa73a","pid":16,"level":30,"msg":"Requiring authentication to web client","time":"2017-08-19T16:23:21.748Z","v":0}
tribeca_1        | {"name":"tribeca:config","hostname":"ec40165fa73a","pid":16,"level":30,"msg":"WebClientListenPort = 3000","time":"2017-08-19T16:23:21.755Z","v":0}
tribe

hnykda

$ for i in $(ls -d */); do PYXECUTOR_LOG_LEVEL=ERROR pyxecutor spss2py -g $i/spss $i; done
20180103 17:32:50:spss2py.parser:ERROR - `Encoding: UTF-8` on lines 1:1 not parsed!
20180103 17:32:50:spss2py.parser:ERROR - `FILTER OFF
USE ALL` on lines 8:9 not parsed!
20180103 17:32:50:spss2py.parser:ERROR - `alter type q4(f20)
alter type q6_1_Count(f20)
alter type q6_2.3.4_Count(f20)
alter type q6_5.6(f20)
alter type h_q6(f20)
alter type q13Ire(f20)

hnykda

dan at think460s in ~/load/FinalWave422122017/tst 
$ ll
total 68M
-rw-r--r-- 1 dan dan 68M Jan  3 19:54 a.hdf
(pygwi) 
dan at think460s in ~/load/FinalWave422122017/tst 
$ python -c "import pandas; df = pandas.read_hdf('a.hdf'); df.to_hdf('b.hdf', 'df', complib='blosc:snappy'); pandas.show_versions()"

INSTALLED VERSIONS
------------------