sse_pub.go

package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

const (
	QueryUrl = "http://query.sse.com.cn/infodisplay/showTradePublicFile.do?isPagination=true&dateTx="
	RefUrl   = "http://www.sse.com.cn/disclosure/diclosure/public/"
)

var date string

func init() {
	flag.StringVar(&date, "date", "2016-12-01", "date for query data. like 2016-12-01")
}

func main() {
	flag.Parse()

	queryUrl := QueryUrl + date

	data, err := Query(queryUrl, RefUrl)
	if err != nil {
		log.Fatal(err)
	}

	body := new(Body)
	if err := json.Unmarshal(data, &body); err != nil {
		log.Fatal(err)
	}

	PrintlnStdout(findBondClass(body.FileContents, len(body.FileContents)))
}

func Query(queryUrl string, refUrl string) ([]byte, error) {
	req, err := http.NewRequest("GET", queryUrl, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Referer", refUrl)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	return data, err
}

func PrintlnStdout(i interface{}) {
	encode := json.NewEncoder(os.Stdout)
	encode.Encode(i)
}

func findBondClass(fileContent []string, contentLen int) []*BondClass {
	bondClasss := make([]*BondClass, 0, 10)
	for i := 0; i < contentLen; i++ {
		// class
		line := getTrimLine(fileContent, i)
		if isIndex(ClassIndexs, line) {
			bondClass := new(BondClass)
			bondClasss = append(bondClasss, bondClass)
			bondClass.Name = getClassName(line)
			if isIndex(SpecialClassIndex, line) {
				bondClass.Bonds, i = findBond(fileContent, contentLen, i+2)
			} else {
				bondClass.BondSubClasss, i = findSubBondClass(fileContent, contentLen, i+1)
			}
		} else {
			continue
		}
	}

	return bondClasss
}

func findSubBondClass(fileContent []string, contentLen int, index int) ([]*BondSubClass, int) {
	firstLine := false
	bondSubClasss := make([]*BondSubClass, 0, 10)
	for i := index; i < contentLen; i++ {
		line := getTrimLine(fileContent, i)
		if !firstLine {
			// 首行为空则无小类
			if doBreak(line) {
				break
			}
			firstLine = true
		}

		if isIndex(SubClassIndexs, line) {
			bondSubClass := new(BondSubClass)
			bondSubClasss = append(bondSubClasss, bondSubClass)
			bondSubClass.Name = getClassName(line)
			bondSubClass.Bonds, i = findBond(fileContent, contentLen, i+1)
		}

		index = i

		if isIndex(ClassIndexs, line) {
			// 触碰到下个大类，则退出
			index--
			break
		}
	}

	return bondSubClasss, index
}

func findBond(fileContent []string, contentLen int, index int) ([]*Bond, int) {
	firstLine := false
	bonds := make([]*Bond, 0, 10)
	for i := index; i < contentLen; i++ {
		line := getTrimLine(fileContent, i)
		if !firstLine {
			// 首行为空则无小类
			if doBreak(line) {
				break
			}
			firstLine = true
		}

		if isIndex(BondIndexs, line) {
			bond := new(Bond)
			parseBond(bond, line)
			bonds = append(bonds, bond)
			index = i
		} else if doBreak(line) {
			break
		}
	}

	for i, _ := range bonds {
		bonds[i].InputSales, index = findSales(fileContent, contentLen, index+1, isInBond)
		bonds[i].OutputSales, index = findSales(fileContent, contentLen, index+1, isOutBond)
	}
	return bonds, index
}

func findSales(fileContent []string, contentLen int, index int, judgeSales func(string) bool) ([]*SalesDepartment, int) {
	firstLine := false
	sales := make([]*SalesDepartment, 0, 10)
	for i := index; i < contentLen; i++ {
		line := getTrimLine(fileContent, i)
		if !firstLine {
			if isIndex(ClassIndexs, line) {
				// 触碰到下个大类，则退出
				// 因为十三比较特殊
				index--
				break
			}

			if !judgeSales(line) {
				continue
			}
			firstLine = true
		}

		if isIndex(BondIndexs, line) {
			sale := new(SalesDepartment)
			parseSale(sale, line)
			sales = append(sales, sale)
		} else if doBreak(line) {
			break
		}
		index = i
	}
	return sales, index
}

// 结束循环规则
func doBreak(line string) bool {
	if line == "" || line == "无" {
		return true
	}
	return false
}

func getTrimLine(fileContent []string, index int) string {
	if index >= len(fileContent) {
		return ""
	} else {
		return strings.TrimSpace(fileContent[index])
	}
}

func getClassName(line string) string {
	names := strings.Split(line, "、")
	if len(names) >= 2 {
		return names[1]
	}
	return line
}

func parseSale(sale *SalesDepartment, line string) {
	contents := SplitAndTrimSpace(line)
	if len(contents) > 2 {
		sale.Name = strings.TrimSpace(contents[1])
		sale.Amount = strings.TrimSpace(contents[len(contents)-1]) + "元"
	} else {
		sale.Name = strings.TrimSpace(line)
	}
}

func parseBond(bond *Bond, line string) {
	// 解析信息，可提取出来封装
	contents := SplitAndTrimSpace(line)
	if len(contents) >= 6 {
		bond.Code = contents[1]
		bond.Name = contents[2]
		bond.Offset = contents[3]
		bond.Deal = contents[4]
		bond.Amount = contents[5] + "万元"
	} else {
		bond.Name = line
	}

	if len(contents) >= 7 {
		bond.Date = contents[6]
	}
	return
}

var (
	SpecialClassIndex = []string{"十一", "十二", "十三", "十四"} // 不生成subclass，而是直接生成bond的class
	ClassIndexs       = []string{"一", "二", "三", "四", "五", "六", "七", "八", "九", "十"}
	SubClassIndexs    = []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"}
	BondIndexs        = []string{"(1", "(2", "(3", "(4", "(5", "(6", "(7", "(8", "(9"}
)

func isIndex(indexs []string, line string) bool {
	for _, index := range indexs {
		if i := strings.Index(line, index); i == 0 {
			goto END
		}
	}

	return false

END:
	return true
}

func isInBond(line string) bool {
	return strings.Contains(line, "买入")
}

func isOutBond(line string) bool {
	return strings.Contains(line, "卖出")
}

func SplitAndTrimSpace(line string) []string {
	contents := strings.Split(line, " ")
	tmp := make([]string, 0, len(contents))
	for _, str := range contents {
		if strings.TrimSpace(str) != "" {
			tmp = append(tmp, str)
		}
	}
	return tmp
}

type Body struct {
	DateTx       string   `json:"dateTx"`
	FileContents []string `json:"fileContents"`
}

type BondClass struct {
	Name          string          `json:",omitempty"`
	Bonds         []*Bond         `json:",omitempty"`
	BondSubClasss []*BondSubClass `json:",omitempty"`
}

type BondSubClass struct {
	Name  string  `json:",omitempty"`
	Bonds []*Bond `json:",omitempty"`
}

type Bond struct {
	Code        string             `json:",omitempty"`
	Name        string             `json:",omitempty"`
	Offset      string             `json:",omitempty"`
	Deal        string             `json:",omitempty"`
	Amount      string             `json:",omitempty"` // 单位万元
	Date        string             `json:",omitempty"`
	InputSales  []*SalesDepartment `json:",omitempty"`
	OutputSales []*SalesDepartment `json:",omitempty"`
}

type SalesDepartment struct {
	Name   string `json:",omitempty"`
	Amount string `json:",omitempty"` // 单位元
}

扒上交所公开交易信息，没仔细研究没项信息生成规则，大致是这样