kemendesa.py

ik = idm.find({}).distinct('id_desa')
key0 = ['Indeks_komposit','Dimensi','skor','Nilai','Perangkat_Indikator','skor_1','Nilai_1','Indikator_peritem','skor_2','kosong','eksisting_dari_indikator_umum','Kewenangan_pusat','Kewenangan_provinsi','Kewenangan_kabupaten','Kewenangan_desa','Kewenangan_sosial_alami']
for u in idmy.find({},{'_id':0}): #sort descending
	if u['id_desa'] not in ik :
		print('mengunduh %s' % u['detail'])
		url0 = root+u['detail'].split('"')[1]
		n=ambil(url)
		kkk=[]
		try:
			for i in n.findAll('div',{'class':'row'})[1].findAll('tr')[2:]:
				kk = [j.text.replace('\n','').strip() for j in i.findAll('td')]
				if len(kk) <16:
					kk = kkk[-1][16:(32-len(kk))] + kk # menyelesaikan permasalahan jumlah kolom yang tidak sama
				kk = kk[:-5] + [j['src'].split('/')[-1].replace('.png','') for j in i.findAll('img')]
				kk = list(u.values())+kk
				kkk.append(kk)
		except:
			kk = list(u.values())
			# kk.insert()
			kkk.append(kk)
		key1 = list(u.keys())+key0 #gabungkan list buat bikin key
		for l in kkk:
			data = dict(zip(key1,l))
			#dt.append(data)
			#if idm.find_one({'id_desa': data['id_desa']},{'id_desa':1,'_id':0}) is None :
			tqdm(idm.insert_one(data))
		ik.append(u['id_desa'])
	else:
		print('data id %s sudah tersedia' % u['id_desa'])