Skip to content

Instantly share code, notes, and snippets.

@vitaminac
Created June 28, 2018 20:18
Show Gist options
  • Save vitaminac/d367da693c2ee1514bef4fe4d3ac6686 to your computer and use it in GitHub Desktop.
Save vitaminac/d367da693c2ee1514bef4fe4d3ac6686 to your computer and use it in GitHub Desktop.
// ==UserScript==
// @name ddcatDebug
// @namespace ddcatDebug
// @description ddcatDebug
// @include *
// @version 1
// @require https://code.jquery.com/jquery-3.2.1.js
// @require https://rawgit.com/kriskowal/q/master/q.js
// @require https://rawgit.com/mgalante/jquery.redirect/master/jquery.redirect.js
// @require https://rawgit.com/ilinsky/jquery-xpath/master/jquery.xpath.js
// @require https://rawgit.com/arantius/3123124/raw/1866c6780e1946f657f688537b199e0102ccd19c/grant-none-shim.js
// @require https://rawgit.com/Olical/EventEmitter/master/EventEmitter.js
// @require http://sited.noear.org/addin/js/cheerio.js
// @grant none
// @run-at document-end
debugger;
// ==/UserScript==
"use strict";
console.log("loading user script ddcatDebug");
const print = console.log;
let globalContext = this;
const callingHistory = [];
// debug use only
function log (functionC, funcname) {
const time = new Date().toLocaleString();
const args = Array.from(arguments).slice(2);
console.log("trace: calling ", typeof functionC === "function" && functionC, funcname || functionC.name, " at ", time, "with argument ", args);
try {
const callHistory = {function: functionC, arguments: args, time: time};
const result = functionC.call(this, ...args);
callHistory.result = result;
callingHistory.push(callHistory);
console.log(functionC.name || funcname, " called result was ", result);
return result;
} catch (e) {
console.log(typeof functionC, functionC, e);
throw e;
}
}
function injectTraceLog (func, contextOpt, propetyname) {
let fc;
const context = contextOpt || globalContext;
if (typeof func === "function") {
fc = (function () {
const permanentContextReference = context;
const logfunc = func;
const name = func.name || propetyname;
let fc;
eval("fc = function " + name + "() {\n" +
" return log.apply(permanentContextReference, [logfunc, name].concat(Array.from(arguments)));\n" +
"};");
return fc;
})();
} else if (typeof func === "string") {
if (typeof context[func] === "function") {
fc = injectTraceLog(context[func], context);
} else {
console.log("can not find ", func, "in current context", context);
}
} else {
console.log("detecting incorrected type of function", func, typeof func);
}
return fc;
}
function copyToContext (exportApi, context) {
try {
for (let publicApi in exportApi) {
if (exportApi.hasOwnProperty(publicApi)) {
context[publicApi] = exportApi[publicApi];
}
}
} catch (e) {
console.log(e);
}
}
function addRequire (src) {
let imported = document.createElement("script");
imported.src = src;
document.head.appendChild(imported);
}
function getPageText () {
let html;
try {
JSON.parse(document.body.firstElementChild.innerHTML);
console.log("document is considered as json type");
html = document.body.firstElementChild.innerHTML;
} catch (e) {
html = document.documentElement.outerHTML;
}
return html;
}
function getCurrentUrl () {
return window.location.toString();
}
function getPageCookieReference () {
return document.cookie;
}
function saveToLocal (objectKey, objectValue) {
GM_setValue(objectKey, JSON.stringify(objectValue));
}
function loadFromLocal (objectKey) {
console.log(__GM_STORAGE_PREFIX);
let obj;
if (GM_listValues().indexOf(objectKey) >= 0) {
obj = JSON.parse(GM_getValue("ddcatConfig"));
}
return obj;
}
const fetch = injectTraceLog(function fetch () {
const deferred = Q.defer();
window.fetch(...Array.from(arguments)).then(function (response) {
response.text().then(function (text) {
// do something with the text response
response.Body = text;
deferred.resolve(response);
}, function (reason) {
console.log(arguments);
deferred.reject(reason);
});
}, function (reason) {
console.log(arguments);
deferred.reject(reason);
});
return deferred.promise;
});
function concatNarrays () {
const arr = Array.from(arguments);
return arr.reduce(function (prev, next) {
if (Array.isArray(next)) {
return prev.concat(next);
} else {
prev.push(next);
return prev;
}
});
}
/* --------------------------------------------------------------------------------------------------- */
console.log("defined global funcition");
function DDCatContainer (importedContext, debug) {
this.context = importedContext;
const self = this;
this._config = {};
Object.defineProperty(this, "config", {
enumerable: true,
configurable: true,
get: this.getCurrentConfig,
set: this.syncCurrentConfig
});
this._configs = {};
this.eventBus = new EventEmitter();
this.eventBus.on("buildNodeConfigCompleted", function () {
self.eventBus.emit("save");
self.eventBus.emit("restart");
});
this.eventBus.on("restart", function () {
self.restart();
self.resume();
});
this.eventBus.on("newInstalledPlugin", function () {
self.syncCurrentConfig();
});
this.eventBus.on("save", function () {
self.saveConfig();
});
if (debug) {
this.enableDebugMode();
}
const exportApi = {
resume: this.resume,
restart: this.restart,
addFunc: this.addFunc,
setDefaultMethod: this.setDefaultMethod,
loadConfigs: this.loadConfigs,
ddcat: this,
setParseMethod: this.setParseMethod,
buildNodeConfig: this.buildNodeConfig,
config: this.config,
save: this.saveConfig,
changeParseMethod: this.changeParseMethod,
installNewPlugin: this.installNewPlugin
};
this.exportApi = Object.assign({}, ...Object.keys(exportApi).map(k => ({[k]: $.isFunction(exportApi[k]) ? exportApi[k].bind(this) : exportApi[k]})));
}
DDCatContainer.prototype.supportParseOptions = ["parse", "parseUrl", "buildUrl", "buildArgs", "buildCookie", "buildRef", "buildHeader"];
DDCatContainer.prototype.exec = function (command) {
this.eventBus.emit(command);
};
DDCatContainer.prototype.saveConfig = function () {
console.log("saving config");
saveToLocal("ddcatConfig", this._configs);
};
DDCatContainer.prototype.loadConfigs = function (configs) {
if (typeof configs === "object") {
Object.assign(this._configs, configs);
} else {
console.log(configs, " is not a valid saved configs file");
}
};
DDCatContainer.prototype.enableDebugMode = function enableDebugMode () {
// trace log each prototype functions
for (let e in this.constructor.prototype) {
if (this.constructor.prototype.hasOwnProperty(e) && typeof this.constructor.prototype[e] === "function" && this.constructor.prototype[e] !== enableDebugMode.name.toString()) {
this[e] = injectTraceLog(this.constructor.prototype[e], this, e);
}
}
};
DDCatContainer.prototype.redirect = function (option, config) {
const method = option.method || config.method || "GET";
if (option.url) {
alert("starts redirection, jump to " + option.url + " with " + option.method + " " + option.body);
try {
$.redirect(option.url, option.body || "", method, "_blank");
} catch (e) {
console.log(e, [option.url, option.body || "", option.method || config.method || "GET", "_blank"]);
}
}
};
DDCatContainer.prototype.parseQueryString = function parseQueryStringb (search) {
let json = {};
search.split(";").forEach(function (pair) {
let [k, v] = pair.split("=");
json[k] = v;
});
return json;
};
DDCatContainer.prototype.getFirstLevelHostName = function () {
let hostname = location.hostname;
return hostname.replace("www.", "");
};
DDCatContainer.prototype.updateCookie = function updateCookie(newCookie, config) {
const cacheThis = this;
this.removeAllCookie(config);
newCookie.split(";").forEach(function (currentValue) {
config.cookies = currentValue + ";path=/;domain=" + cacheThis.getFirstLevelHostName();
});
};
DDCatContainer.prototype.removeAllCookie = function (config) {
const cacheThis = this;
config.cookies.split(";").forEach(function (currentValue, index, array) {
config.cookies = currentValue + ";expires=" + new Date().toGMTString() + ";path=/;domain=" + cacheThis.getFirstLevelHostName();
});
};
DDCatContainer.prototype.prepareNewRequsts = function prepareNewRequsts (urlCallSet, config) {
const cacheThis = this;
return urlCallSet.map(function (callUrl) {
const parseUrlResult = callUrl.split("::");
const request = {};
const header = new Headers();
let newMethod = "";
let newUrl = callUrl;
let args = "";
if (parseUrlResult.length > 1) {
newUrl = parseUrlResult[parseUrlResult.length - 1];
if (parseUrlResult.length > 2) {
newMethod = parseUrlResult[parseUrlResult.length - 2];
}
} else {
newUrl = parseUrlResult[0];
}
if (config.buildUrl) {
newUrl = config.buildUrl(newUrl);
}
if (config.buildArgs) {
const newArgs = config.buildArgs(newUrl);
if (newArgs) {
args += newArgs;
// args = parseQueryString(args);
// request.body = JSON.stringify(args);
request.body = args.replace(/;/g, "&");
}
}
if (config.buildRefer) {
header.set("Referer", config.buildRefer(newUrl));
}
if (config.buildHeader) {
let k, v;
let newHeader = config.buildHeader(newUrl);
if (!newHeader.includes("$$")) {
newHeader.replace(/;/g, "$$$$");
newHeader = newHeader.replace(/=/g, ":");
}
newHeader.split("$$").forEach(function (x) {
if (x) {
[k, v] = x.split(":");
header.append(k.trim(), (v && v.trim()) || "");
} else {
console.log("warning header is null and buildHeader is enable");
}
});
}
if (config.buildCookie) {
let cookie = config.buildCookie(newUrl, config.cookies);
cacheThis.updateCookie(cookie, config);
request.credentials = "same-origin";
}
request.url = newUrl;
request.method = newMethod || config.method;
request.headers = header;
return request;
});
};
DDCatContainer.prototype.parseUrl = function (url, html, config) {
const cacheThis = this;
let promises;
let parseUrlResultSet;
let parseUrlResult;
let deferred = Q.defer();
if (config.parseUrl) {
parseUrlResult = config.parseUrl(url, html);
if (parseUrlResult) {
parseUrlResultSet = parseUrlResult.split(";");
}
}
if (parseUrlResultSet && (parseUrlResultSet.length > 1 || parseUrlResultSet[0] !== url)) {
promises = this.prepareNewRequsts(parseUrlResultSet, config).map(function (option, index) {
return fetch(option.url, option).then(function (response) {
if (parseUrlResultSet[index].toUpperCase().startsWith("CALL")) {
return cacheThis.parseUrl(response.url, response.Body, config);
} else {
return [{url: response.url, body: response.Body}];
}
}, function (reason) {
console.log(reason);
return [{url: url, body: html}];
});
});
}
if (promises && Array.isArray(promises) && promises.length > 0) {
Q.allSettled(promises).then(function (results) {
const r = concatNarrays(...results.map(function (result) {
if (result.state === "fulfilled") {
return result.value;
} else {
console.log(result);
}
}).filter(function (result) {
return result;
}));
deferred.resolve(r);
});
} else {
deferred.resolve([{url: url, body: html}]);
}
return deferred.promise;
};
DDCatContainer.prototype.firstStep = function parseResult (config) {
const cacheThis = this;
this.parseUrl(config.url, config.html, config)
.then(function (results) {
let listObj = [];
results.forEach(function (result) {
try {
const obj = JSON.parse(config.parse(result.url, result.body));
if (Array.isArray(obj)) {
listObj = listObj.concat(obj);
} else {
listObj.push(obj);
}
} catch (e) {
console.log(e);
}
});
return listObj;
}, function (reason) {
console.log(reason, "something went wrong at parseUrl");
throw reason;
})
.then(function (listOfObj) {
console.log(listOfObj);
let nextStep = cacheThis.jumpToTarget.bind(cacheThis, cacheThis.config, listOfObj);
cacheThis.eventBus.off("jump");
cacheThis.eventBus.once("jump", nextStep);
}).done();
};
DDCatContainer.prototype.jumpToTarget = function finalStepd (config, listOfObj) {
const cacheThis = this;
let option;
try {
listOfObj.forEach(function (obj) {
if (typeof obj === "string") {
option = {url: obj};
cacheThis.redirect(option, config);
} else {
if (obj.url) {
option = {url: obj.url};
cacheThis.redirect(option, config);
}
if (obj.sections) {
cacheThis.jumpToTarget(config, obj.sections);
}
}
});
} catch (e) {
console.log(e, option);
throw e;
}
};
DDCatContainer.prototype.resume = function () {
this.eventBus.emit("resume");
};
DDCatContainer.prototype.addFunc = function (func, functionName) {
if (typeof func === "function") {
this[func.name || functionName] = func;
}
};
DDCatContainer.prototype.restart = function () {
if (typeof this.firstStep === "function") {
const cacheThis = this;
this.eventBus.once("resume", function () {
cacheThis.firstStep(cacheThis.config);
});
} else {
console.log(this.firstStep, " is not a function");
}
};
DDCatContainer.prototype.setDefaultMethod = function (method) {
this.config.method = method;
};
DDCatContainer.prototype.setParseMethod = function (parserType, parser) {
console.log("you are setting ", parserType, " as ", parser);
if ((typeof parser === "function") || (typeof parser === "string")) {
return injectTraceLog(globalContext[parser] || eval("(function(){return " + parser + "})()"), this);
} else {
console.log("wrong function ", parser);
}
};
DDCatContainer.prototype.changeParseMethod = function (parserType, parser) {
this.config[parserType] = injectTraceLog(parser, this);
};
DDCatContainer.prototype.installNewPlugin = function (plugin) {
const xmlDoc = jQuery.parseXML(plugin);
const $xml = $(xmlDoc);
const title = $xml.xpath("//title[position()=1]").text();
const expr = $xml.xpath("//expr[position()=1]").text();
this.tryLoadPluginJSCode($xml);
this._configs[title] = {
sited: plugin,
expr: expr
};
this.eventBus.emit("newInstalledPlugin");
};
DDCatContainer.prototype.tryLoadPluginJSCode = function ($plugin) {
const cacheThis = this;
try {
const code = $plugin.xpath("(/site/(jscript | script))[position()=1]").text().trim();
try {
eval.call(cacheThis, code.replace(/"\s*use\s+strict\s*"\s*;/g, "")); // 认不出 strict
console.log("finded jscode", code);
} catch (e) {
console.log("javascript run-time compile error");
throw e;
}
} catch (e) {
console.log(e);
throw {message: "插件不是正确的格式"};
}
};
DDCatContainer.prototype.updateCurrentConfig = function (newConfig) {
if (newConfig) {
Object.assign(this._config || {}, newConfig);
}
};
DDCatContainer.prototype.getCurrentConfig = function () {
if ((Object.keys(this._config).length === 0)) {
this.syncCurrentConfig();
}
return this._config;
};
DDCatContainer.prototype.syncCurrentConfig = function () {
let config = this.matchCurrentSite(getCurrentUrl(), this._configs);
if (config) {
this.updateCurrentConfig(config);
this.eventBus.emit("buildNodeConfigCompleted");
}
};
DDCatContainer.prototype.matchCurrentSite = function (cuurentUrl, configs) {
for (let e in configs) {
if (configs.hasOwnProperty(e) && configs[e].expr) {
try {
let re = new RegExp(this._configs[e].expr, "i");
if (cuurentUrl.match(re)) {
let xmlDoc = $.parseXML(this._configs[e].sited);
let $xml = $(xmlDoc);
console.log("matchs ", e);
this.tryLoadPluginJSCode($xml);
return this.generateConfig($xml);
}
console.log(re);
} catch (e) {
console.log(e);
}
}
}
};
DDCatContainer.prototype.generateConfig = function ($xml) {
const expr = $xml.xpath("/site/main//*[@url or @expr]");
const listOfMatch = [];
expr.each(function (i, e) {
let re;
if ($(e).attr("url")) {
re = new RegExp("^" + $(e).attr("url").replace(/&/g, "&").replace(/([.\\/?])/g, "\\$1").replace("@page", "\\d+").replace("@key", ".*") + "/*$", "i");
} else if ($(e).attr("expr")) {
re = new RegExp($(e).attr("expr"));
} else {
console.log(e, "does'n seem to be a right NodeSet");
}
if (window.location.toString().match(re)) {
listOfMatch.push({regex: re, element: e});
}
});
console.log("this sited could be one of the following ", listOfMatch);
let maxItem = {length: 0};
$.each(listOfMatch, function (i, item) {
if (item.regex.toString().length > maxItem.length) {
maxItem = item;
maxItem.length = item.regex.toString().length;
}
});
console.log("current node position is", maxItem);
if (maxItem.element) {
return this.buildNodeConfig(maxItem.element);
}
};
DDCatContainer.prototype.buildNodeConfig = function (NodeSet) {
const cacheThis = this;
const currentConfig = {};
$(NodeSet).xpath("ancestor-or-self::*").each(function (i, parent) {
Object.assign(currentConfig, cacheThis.generateAttrNode(parent));
});
currentConfig.subConfigs = this.buildChildrenConfig(NodeSet);
if (currentConfig) {
currentConfig.cookies = getPageCookieReference();
currentConfig.url = getCurrentUrl();
currentConfig.html = getPageText();
console.log(currentConfig);
} else {
console.log("some error had happened while parsing the config of ", NodeSet);
}
return currentConfig;
};
DDCatContainer.prototype.generateAttrNode = function (currentNode) {
const cacheThis = this;
console.log(currentNode);
let attributeSet;
if (currentNode) {
attributeSet = {};
Array.from($(currentNode).xpath("@*")).forEach(function (x) {
try {
attributeSet[x.nodeName] = $(x).val();
} catch (e) {
console.log(e);
}
console.log(attributeSet);
});
}
this.supportParseOptions.forEach(function (parserName) {
if (attributeSet[parserName]) {
attributeSet[parserName] = cacheThis.setParseMethod(parserName, attributeSet[parserName]);
}
});
return attributeSet;
};
DDCatContainer.prototype.buildChildrenConfig = function (currentNode) {
const cacheThis = this;
const subConfigs = [];
let childrenElements = Array.from($(currentNode).xpath("child::*"));
console.log(childrenElements);
if (childrenElements && Array.isArray(childrenElements) && childrenElements.length > 0) {
childrenElements.forEach(function (e) {
let subConfig = cacheThis.generateAttrNode(e);
if (subConfig) {
subConfigs.push(subConfig);
}
});
console.log("children elements were ", subConfigs);
}
return subConfigs;
};
console.log("container' Constructor defined");
// here we go
// if return json data, be sure that firefox's config devtools.jsonview.enabled is set to false;
function initialize () {
// $.noConflict(true);
const ddContainer = new DDCatContainer(window, true);
const exportApi = Object.assign(ddContainer.exportApi, {
cheerio: cheerio, Q: Q, $jQuery: $, globalContext: globalContext, callingHistory: callingHistory
});
const createButton = function (callback) {
let inputPosition = {
"position": "absolute",
"top": "0",
"right": "0",
"width": "100%",
"height": "100%",
"display": "block"
};
let containerStyle = {
"width": "50px",
"height": "50px",
"position": "fixed",
"top": "0",
"right": "0",
"z-index": "999"
};
let $container = $("<div>").css(containerStyle);
let $imgInput = $("<input>", {
type: "image",
src: "https://cdn2.iconfinder.com/data/icons/music-bento/100/stop-512.png",
name: "load"
}).css(inputPosition).appendTo($container);
let $dUploader = $("<input>", {
type: "file",
accept: ".xml,.sited"
}).css(inputPosition).css({
"opacity": "0",
"z-index": "2"
}).change(function () {
if ($(this).val()){
let fileName = $(this).val();
let file = this.files[0];
let fr = new FileReader();
fr.onload = function () {
callback(fr.result);
};
fr.readAsText(file);
console.log("reading from ", fileName);
$dUploader.val("");
}
}).appendTo($container);
$("body").append($container);
};
copyToContext(exportApi, window);
createButton(exportApi.installNewPlugin);
window.onbeforeunload = function () {
exportApi.save();
};
window.onload = function () {
console.log("try loading previous saved config to ", exportApi.config);
let loadedConfigs = loadFromLocal("ddcatConfig");
console.log("loaded ", loadedConfigs);
exportApi.loadConfigs(loadedConfigs);
console.log("success loaded, current page config is ", ddContainer.config);
};
}
$(document).ready(function () {
initialize();
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment