Skip to content

Instantly share code, notes, and snippets.

View btnguyen2k's full-sized avatar

Thanh Ba Nguyen btnguyen2k

  • Microsoft
  • Sydney, Australia
View GitHub Profile
@btnguyen2k
btnguyen2k / OpenAINormalizeAndEmbeddings.go
Created May 15, 2023 04:54
Normalize input text for embeddings or not?
package main
import (
"fmt"
"os"
"regexp"
"strings"
"github.com/btnguyen2k/oaiaux"
)
@btnguyen2k
btnguyen2k / CountOpenAITokens.go
Created May 11, 2023 13:49
Count OpenAI tokens in different encodings
package main
import (
"fmt"
"github.com/btnguyen2k/oaiaux"
)
func CountTokens(text, lang, encoding string) {
numTokens := oaiaux.CountTokens(text, oaiaux.Option{Key: "encoding", Value: encoding})
@btnguyen2k
btnguyen2k / oai_esttokens.go
Created May 3, 2023 08:40
Estimate the number of tokens for an OpenAI prompt.
// EstimateTokens estimates the number of tokes for an input string.
func EstimateTokens(input string) int {
const re1 = `[^\w\d]+`
const re2 = `[\w\d]+`
reWords := regexp.MustCompile(re1)
words := reWords.Split(input, -1)
numWords := 0
for _, w := range words {
if w != "" {
numBytes := len([]byte(w))
@btnguyen2k
btnguyen2k / aks_change_system_reserved_resources.yaml
Last active July 9, 2020 12:27
(#AKSHack change AKS reserved memory)
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kured
rules:
# Allow kured to read spec.unschedulable
# Allow kubectl to drain/uncordon
#
# NB: These permissions are tightly coupled to the bundled version of kubectl; the ones below
import java.util.*;
/**
* A demo Java application to see how GC works.
*
* @author Thanh Nguyen
*/
public class GcMemDemo {
static Map<Long, byte[]> buffer = new HashMap<>();
#!/bin/bash
echo '=================================================='
echo '===============Install necessary tools==============='
echo '=================================================='
sudo yum update
sudo yum install git make flex bison libtool automake openssl-devel libevent libevent-devel python-devel gcc-c++ byacc java-1.7.0-openjdk ant
echo '=================================================='
@btnguyen2k
btnguyen2k / DPathUtils.java
Created September 13, 2012 18:27
Utility to get from a Java Map/List/array using "path" expression
package ddth.dasp.framework.utils;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class DPathUtils {
private final static Pattern PATTERN_INDEX = Pattern