Skip to content

Instantly share code, notes, and snippets.

# -*- coding: utf8 -*-
from datetime import datetime
from libs.common.Date import Date
import logging
from app import redis
from app.pack import Pack
from app.page import Page
from app.tasks.parse import parse_content_wrapper
import config
# -*- coding: utf8 -*-
from datetime import datetime, timedelta
import json
from libs.common.Date import Date
from libs.common.os_tools import create_file, remove_file, create_gzip, \
read_gzip
from libs.common.string import md5, encode_from_page_charset, hash_string
import logging
import os
# -*- coding: utf8 -*-
from datetime import datetime
from libs.common.Date import Date
import logging
import time
from app import redis
from app.redis_keys import SITE_PACK_PREFIX, URL_QUEUE, SITE_QUEUE_LEN_PREFIX, \
SITE_LAST_DOWNLOAD_PREFIX, PACK_META_PREFIX
from app.site import Site, NoPacks
# -*- coding: utf8 -*-
from gevent import monkey
from app.pack import Pack
monkey.patch_socket()
monkey.patch_ssl()
monkey.patch_thread()
from app import redis
from app.page import Page
@askabelin
askabelin / read_async_get.py
Last active April 12, 2018 08:44
Multiple readers work asynchronously
import asyncio
import aio_pika
async def read(reader_id):
connection = await aio_pika.connect_robust(**RMQ_CONFIG)
async with connection:
channel = await connection.channel()
queue = await channel.declare_queue('test_queue', durable=True)
message = await queue.get()
@askabelin
askabelin / read_async_iter.py
Last active April 11, 2018 16:19
Multiple readers work synchronously
import asyncio
import aio_pika
async def read(reader_id):
connection = await aio_pika.connect_robust(**RMQ_CONFIG)
async with connection:
channel = await connection.channel()
queue = await channel.declare_queue('test_queue', durable=True)
message = await queue.get()