Skip to content

Instantly share code, notes, and snippets.

@alper
Created April 18, 2012 16:25
Show Gist options
  • Save alper/2414715 to your computer and use it in GitHub Desktop.
Save alper/2414715 to your computer and use it in GitHub Desktop.
Thinkup Tweet importetr
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
// Generated on: 2012-04-13 23:44:46 GMT+00:00
// ************** Tweet 1 of 47455 **************
{
"in_reply_to_screen_name": null,
"retweeted": false,
"created_at": "Sat Aug 18 22:56:52 +0000 2007",
"id_str": "213346122",
"geo": null,
"entities": {
"user_mentions": [
],
"hashtags": [
],
"urls": [
]
},
"source": "web",
"in_reply_to_status_id_str": null,
"favorited": false,
"in_reply_to_user_id": null,
"contributors": null,
"coordinates": null,
"user": {
"id_str": "8273852",
"id": 526752675267
},
"id": 213346122,
"retweet_count": 0,
"truncated": false,
"text": "Finally joined Twitter. Working at Lowlands for VPRO 3voor12. Just finished my last photos: M.I.A. was great!",
"in_reply_to_user_id_str": null,
"place": null,
"in_reply_to_status_id": null
}
// ************** Tweet 2 of 47455 **************
{
"in_reply_to_screen_name": null,
"retweeted": false,
"created_at": "Sun Aug 19 09:56:41 +0000 2007",
"id_str": "213973732",
"geo": null,
"entities": {
"user_mentions": [
],
"hashtags": [
{
"indices": [
0,
5
],
"text": "LL07"
}
],
"urls": [
]
},
"source": "web",
"in_reply_to_status_id_str": null,
"favorited": false,
"in_reply_to_user_id": null,
"contributors": null,
"coordinates": null,
"user": {
"id_str": "8273852",
"id": 526752675267
},
"id": 213973732,
"retweet_count": 0,
"truncated": false,
"text": "#LL07 wakker worden in het 3voor12 redactiehok en straks bandjes verdelen voor de foto's. nu al zin in aux raus.",
"in_reply_to_user_id_str": null,
"place": null,
"in_reply_to_status_id": null
}
// ************** Tweet 3 of 47455 **************
{
"in_reply_to_screen_name": null,
"retweeted": false,
"created_at": "Sun Aug 19 10:32:29 +0000 2007",
"possibly_sensitive": false,
"id_str": "214005832",
"geo": null,
"entities": {
"user_mentions": [
],
"hashtags": [
{
"indices": [
0,
5
],
"text": "LL07"
}
],
"urls": [
{
"expanded_url": null,
"indices": [
45,
114
],
"url": "http://www.flickr.com/photos/silvertje/collections/72157601507584668/"
}
]
},
"source": "web",
"in_reply_to_status_id_str": null,
"favorited": false,
"in_reply_to_user_id": null,
"contributors": null,
"coordinates": null,
"user": {
"id_str": "8273852",
"id": 526752675267
},
"id": 214005832,
"retweet_count": 0,
"truncated": false,
"text": "#LL07 Nog meer Lowlands (3voor12) foto's op: http://www.flickr.com/photos/silvertje/collections/72157601507584668/",
"in_reply_to_user_id_str": null,
"place": null,
"in_reply_to_status_id": null
}
<?php
chdir(dirname(__FILE__) . '/../..');
require_once 'init.php';
//Avoid "Error: DateTime::__construct(): It is not safe to rely on the system's timezone settings" error
require_once 'config.inc.php';
date_default_timezone_set($THINKUP_CFG['timezone']);
// don't run via the web...
if (isset($_SERVER['SERVER_NAME'])) {
die("This script should only be run via the command line.");
}
try {
// do we need a migration?
$db_version = UpgradeDatabaseController::getCurrentDBVersion($cached = false);
$config = Config::getInstance();
$thinkup_db_version = $config->getValue('THINKUP_VERSION');
// Hacking in here my import code
// Loader::addPath('/home/alper/thinkuptest.aardverschuiving.com/plugins/twitter/model/');
//
// $instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
// $plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
// $options = $plugin_option_dao->getOptionsHash('twitter', true);
// $instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('twitter');
// $instance = $instances[0];
//
// $crawler = new TwitterCrawler($instance, NULL);
//
// var_dump($crawler);
$post_dao = DAOFactory::getDAO('PostDAO');
// Modify this path
$file = fopen('/home/alper/thinkuptest.aardverschuiving.com/webapp/install/cli/web_martin-tweets.txt', 'r');
$started = false;
$tweet_lines = array();
while(True) {
$line = fgets($file);
if (strpos($line, '// *****') !== FALSE || feof($file)) {
// This is the tweet marker, it can mark the beginning or the end of a tweet
// echo $line;
$started = true;
if (count($tweet_lines) > 0) {
// There is a tweet here, the marker marked the end of a tweet
$tweettext = join(' ', $tweet_lines);
// echo $tweettext;
$json = json_decode($tweettext);
// var_dump($tweet);
$parsed_tweet = array(
'post_id' => $json->id,
'author_username' => 'alper', // Hardcoded
'author_fullname' => 'Alper Cugun', // Hardcoded
'author_avatar' => 'https://twimg0-a.akamaihd.net/profile_images/1095412796/AlperAVA3.jpg', // Hardcoded
'is_protected' => 0, // Hardcoded
'author_user_id' => (string)$json->user->id,
'user_id' => (string)$json->user->id,
'post_text' => (string)$json->text,
'pub_date' => gmdate("Y-m-d H:i:s", strToTime($json->created_at)),
'in_reply_to_post_id' => (string)$json->in_reply_to_status_id,
'in_reply_to_user_id' => (string)$json->in_reply_to_user_id,
'source' => (string)$json->source,
'favorited' => (string)$json->favorited,
'place' => (string)$json->place->full_name,
'network' => 'twitter'
);
// echo 'parsed tweet' . $parsed_tweet['post_id'] . ' ';
$inserted_post_key = $post_dao->addPost($parsed_tweet);
echo 'key ' . $inserted_post_key . ' ';
// Empty the lines for the next tweet
$tweet_lines = array();
}
if (feof($file)) break;
} else if ($started) {
// echo count($tweet_lines) . ' ';
array_push($tweet_lines, $line);
}
}
exit;
} catch(Exception $e) {
error_log(" Error: " . $e->getMessage() . "\n");
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment