Plugins support (#46)

* modify * update * add acl * add feature * update dockerfile * add deploy * update * update * plugins * plugins * update * update * update * fixed * remove * fixed * add log * update * fixed * update * fix config * add http api * add http api * resp * add config for work chan * update * fixed * update * disable trace * fixed * change acl * fixed * fixed res * dd * dd * ddd * dd * update * fixed * update * add * fixed * update key * add log * update * format * update * update auth * update * update readme * added * update * fixed * fixed * fix * upade * update * update
2026-08-30 14:24:53 +00:00 · 2019-07-25 13:54:42 +08:00
parent daf4a0e0f5
commit c6b1f1db42
770 changed files with 213077 additions and 2659 deletions
@@ -3,79 +3,41 @@
 package broker

 import (
-	"github.com/fhmq/hmq/lib/acl"
-	"github.com/fsnotify/fsnotify"
-	"go.uber.org/zap"
 	"strings"
+
+	"github.com/fhmq/hmq/plugins/authhttp"
 )

 const (
-	PUB = 1
-	SUB = 2
+	SUB = "1"
+	PUB = "2"
 )

-func (c *client) CheckTopicAuth(typ int, topic string) bool {
-	if c.typ != CLIENT || !c.broker.config.Acl {
-		return true
+func (b *Broker) CheckTopicAuth(action, username, topic string) bool {
+	if b.pluginAuthHTTP {
+		if strings.HasPrefix(topic, "$SYS/broker/connection/clients/") {
+			return true
+		}
+
+		if strings.HasPrefix(topic, "$queue/") {
+			topic = strings.TrimPrefix(topic, "$queue/")
+		}
+
+		return authhttp.CheckACL(username, action, topic)
 	}
-	if strings.HasPrefix(topic, "$queue/") {
-		topic = string([]byte(topic)[7:])
-		if topic == "" {
+
+	return true
+
+}
+
+func (b *Broker) CheckConnectAuth(clientID, username, password string) bool {
+	if b.pluginAuthHTTP {
+		if clientID == "" || username == "" {
 			return false
 		}
+		return authhttp.CheckAuth(clientID, username, password)
 	}
-	ip := c.info.remoteIP
-	username := string(c.info.username)
-	clientid := string(c.info.clientID)
-	aclInfo := c.broker.AclConfig
-	return acl.CheckTopicAuth(aclInfo, typ, ip, username, clientid, topic)
+
+	return true

 }
-
-var (
-	watchList = []string{"./conf"}
-)
-
-func (b *Broker) handleFsEvent(event fsnotify.Event) error {
-	switch event.Name {
-	case b.config.AclConf:
-		if event.Op&fsnotify.Write == fsnotify.Write ||
-			event.Op&fsnotify.Create == fsnotify.Create {
-			log.Info("text:handling acl config change event:", zap.String("filename", event.Name))
-			aclconfig, err := acl.AclConfigLoad(event.Name)
-			if err != nil {
-				log.Error("aclconfig change failed, load acl conf error: ", zap.Error(err))
-				return err
-			}
-			b.AclConfig = aclconfig
-		}
-	}
-	return nil
-}
-
-func (b *Broker) StartAclWatcher() {
-	go func() {
-		wch, e := fsnotify.NewWatcher()
-		if e != nil {
-			log.Error("start monitor acl config file error,", zap.Error(e))
-			return
-		}
-		defer wch.Close()
-
-		for _, i := range watchList {
-			if err := wch.Add(i); err != nil {
-				log.Error("start monitor acl config file error,", zap.Error(err))
-				return
-			}
-		}
-		log.Info("watching acl config file change...")
-		for {
-			select {
-			case evt := <-wch.Events:
-				b.handleFsEvent(evt)
-			case err := <-wch.Errors:
-				log.Error("error:", zap.Error(err))
-			}
-		}
-	}()
-}
@@ -10,13 +10,15 @@ import (
 	_ "net/http/pprof"
 	"runtime/debug"
 	"sync"
-	"sync/atomic"
 	"time"

+	"github.com/fhmq/hmq/broker/lib/sessions"
+	"github.com/fhmq/hmq/broker/lib/topics"
+	"github.com/fhmq/hmq/plugins"
+
 	"github.com/eclipse/paho.mqtt.golang/packets"
-	"github.com/fhmq/hmq/lib/acl"
-	"github.com/fhmq/hmq/lib/sessions"
-	"github.com/fhmq/hmq/lib/topics"
+	"github.com/fhmq/hmq/plugins/authhttp"
+	"github.com/fhmq/hmq/plugins/kafka"
 	"github.com/fhmq/hmq/pool"
 	"github.com/shirou/gopsutil/mem"
 	"go.uber.org/zap"
@@ -34,22 +36,20 @@ type Message struct {
 }

 type Broker struct {
-	id          string
-	cid         uint64
-	mu          sync.Mutex
-	config      *Config
-	tlsConfig   *tls.Config
-	AclConfig   *acl.ACLConfig
-	wpool       *pool.WorkerPool
-	clients     sync.Map
-	routes      sync.Map
-	remotes     sync.Map
-	nodes       map[string]interface{}
-	clusterPool chan *Message
-	queues      map[string]int
-	topicsMgr   *topics.Manager
-	sessionMgr  *sessions.Manager
-	// messagePool []chan *Message
+	id             string
+	mu             sync.Mutex
+	config         *Config
+	tlsConfig      *tls.Config
+	wpool          *pool.WorkerPool
+	clients        sync.Map
+	routes         sync.Map
+	remotes        sync.Map
+	nodes          map[string]interface{}
+	clusterPool    chan *Message
+	topicsMgr      *topics.Manager
+	sessionMgr     *sessions.Manager
+	pluginAuthHTTP bool
+	pluginKafka    bool
 }

 func newMessagePool() []chan *Message {
@@ -67,7 +67,6 @@ func NewBroker(config *Config) (*Broker, error) {
 		config:      config,
 		wpool:       pool.New(config.Worker),
 		nodes:       make(map[string]interface{}),
-		queues:      make(map[string]int),
 		clusterPool: make(chan *Message),
 	}

@@ -92,15 +91,18 @@ func NewBroker(config *Config) (*Broker, error) {
 		}
 		b.tlsConfig = tlsconfig
 	}
-	if b.config.Acl {
-		aclconfig, err := acl.AclConfigLoad(b.config.AclConf)
-		if err != nil {
-			log.Error("Load acl conf error", zap.Error(err))
-			return nil, err
+
+	for _, plugin := range b.config.Plugins {
+		switch plugin {
+		case authhttp.AuthHTTP:
+			authhttp.Init()
+			b.pluginAuthHTTP = true
+		case kafka.Kafka:
+			kafka.Init()
+			b.pluginKafka = true
 		}
-		b.AclConfig = aclconfig
-		b.StartAclWatcher()
 	}
+
 	return b, nil
 }

@@ -125,6 +127,8 @@ func (b *Broker) Start() {
 		return
 	}

+	go InitHTTPMoniter(b)
+
 	//listen clinet over tcp
 	if b.config.Port != "" {
 		go b.StartClientListening(false)
@@ -154,10 +158,9 @@ func (b *Broker) Start() {
 	//system monitor
 	go StateMonitor()

-	if b.config.Debug {
-		startPProf()
-	}
-
+	// if b.config.Debug {
+	// 	startPProf()
+	// }
 }

 func startPProf() {
@@ -198,7 +201,6 @@ func (b *Broker) StartWebsocketListening() {

 func (b *Broker) wsHandler(ws *websocket.Conn) {
 	// io.Copy(ws, ws)
-	atomic.AddUint64(&b.cid, 1)
 	ws.PayloadType = websocket.BinaryFrame
 	b.handleConnection(CLIENT, ws)
 }
@@ -238,7 +240,6 @@ func (b *Broker) StartClientListening(Tls bool) {
 			continue
 		}
 		tmpDelay = ACCEPT_MIN_SLEEP
-		atomic.AddUint64(&b.cid, 1)
 		go b.handleConnection(CLIENT, conn)

 	}
@@ -323,17 +324,46 @@ func (b *Broker) handleConnection(typ int, conn net.Conn) {
 		return
 	}

-	log.Info("reconnect connect from ", zap.String("clientID", msg.ClientIdentifier))
+	log.Info("read connect from ", zap.String("clientID", msg.ClientIdentifier))

 	connack := packets.NewControlPacket(packets.Connack).(*packets.ConnackPacket)
-	connack.ReturnCode = packets.Accepted
 	connack.SessionPresent = msg.CleanSession
+	connack.ReturnCode = msg.Validate()
+
+	if connack.ReturnCode != packets.Accepted {
+		err = connack.Write(conn)
+		if err != nil {
+			log.Error("send connack error, ", zap.Error(err), zap.String("clientID", msg.ClientIdentifier))
+			return
+		}
+		return
+	}
+
+	if typ == CLIENT && !b.CheckConnectAuth(string(msg.ClientIdentifier), string(msg.Username), string(msg.Password)) {
+		connack.ReturnCode = packets.ErrRefusedNotAuthorised
+		err = connack.Write(conn)
+		if err != nil {
+			log.Error("send connack error, ", zap.Error(err), zap.String("clientID", msg.ClientIdentifier))
+			return
+		}
+		return
+	}
+
 	err = connack.Write(conn)
 	if err != nil {
 		log.Error("send connack error, ", zap.Error(err), zap.String("clientID", msg.ClientIdentifier))
 		return
 	}

+	if typ == CLIENT {
+		b.Publish(&plugins.Elements{
+			ClientID:  string(msg.ClientIdentifier),
+			Username:  string(msg.Username),
+			Action:    plugins.Connect,
+			Timestamp: time.Now().Unix(),
+		})
+	}
+
 	willmsg := packets.NewControlPacket(packets.Publish).(*packets.PublishPacket)
 	if msg.WillFlag {
 		willmsg.Qos = msg.WillQos
@@ -397,8 +427,6 @@ func (b *Broker) handleConnection(typ int, conn net.Conn) {
 		b.routes.Store(cid, c)
 	}

-	// mpool := b.messagePool[fnv1a.HashString64(cid)%MessagePoolNum]
-
 	c.readLoop()
 }

@@ -5,15 +5,19 @@ package broker
 import (
 	"context"
 	"errors"
+	"math/rand"
 	"net"
 	"reflect"
+	"regexp"
 	"strings"
 	"sync"
 	"time"

+	"github.com/fhmq/hmq/broker/lib/sessions"
+	"github.com/fhmq/hmq/broker/lib/topics"
+	"github.com/fhmq/hmq/plugins"
+
 	"github.com/eclipse/paho.mqtt.golang/packets"
-	"github.com/fhmq/hmq/lib/sessions"
-	"github.com/fhmq/hmq/lib/topics"
 	"go.uber.org/zap"
 )

@@ -28,34 +32,45 @@ const (
 	REMOTE  = 2
 	CLUSTER = 3
 )
+
+const (
+	_GroupTopicRegexp = `^\$share/([0-9a-zA-Z_-]+)/(.*)$`
+)
+
 const (
 	Connected    = 1
 	Disconnected = 2
 )

+var (
+	groupCompile = regexp.MustCompile(_GroupTopicRegexp)
+)
+
 type client struct {
-	typ        int
-	mu         sync.Mutex
-	broker     *Broker
-	conn       net.Conn
-	info       info
-	route      route
-	status     int
-	ctx        context.Context
-	cancelFunc context.CancelFunc
-	session    *sessions.Session
-	subMap     map[string]*subscription
-	topicsMgr  *topics.Manager
-	subs       []interface{}
-	qoss       []byte
-	rmsgs      []*packets.PublishPacket
+	typ         int
+	mu          sync.Mutex
+	broker      *Broker
+	conn        net.Conn
+	info        info
+	route       route
+	status      int
+	ctx         context.Context
+	cancelFunc  context.CancelFunc
+	session     *sessions.Session
+	subMap      map[string]*subscription
+	topicsMgr   *topics.Manager
+	subs        []interface{}
+	qoss        []byte
+	rmsgs       []*packets.PublishPacket
+	routeSubMap map[string]uint64
 }

 type subscription struct {
-	client *client
-	topic  string
-	qos    byte
-	queue  bool
+	client    *client
+	topic     string
+	qos       byte
+	share     bool
+	groupName string
 }

 type info struct {
@@ -75,6 +90,7 @@ type route struct {

 var (
 	DisconnectdPacket = packets.NewControlPacket(packets.Disconnect).(*packets.DisconnectPacket)
+	r                 = rand.New(rand.NewSource(time.Now().UnixNano()))
 )

 func (c *client) init() {
@@ -104,13 +120,21 @@ func (c *client) readLoop() {
 			//add read timeout
 			if err := nc.SetReadDeadline(time.Now().Add(timeOut)); err != nil {
 				log.Error("set read timeout error: ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+				msg := &Message{
+					client: c,
+					packet: DisconnectdPacket,
+				}
+				b.SubmitWork(c.info.clientID, msg)
 				return
 			}

 			packet, err := packets.ReadPacket(nc)
 			if err != nil {
 				log.Error("read packet error: ", zap.Error(err), zap.String("ClientID", c.info.clientID))
-				msg := &Message{client: c, packet: DisconnectdPacket}
+				msg := &Message{
+					client: c,
+					packet: DisconnectdPacket,
+				}
 				b.SubmitWork(c.info.clientID, msg)
 				return
 			}
@@ -131,7 +155,11 @@ func ProcessMessage(msg *Message) {
 	if ca == nil {
 		return
 	}
-	log.Debug("Recv message:", zap.String("message type", reflect.TypeOf(msg.packet).String()[9:]), zap.String("ClientID", c.info.clientID))
+
+	if c.typ == CLIENT {
+		log.Debug("Recv message:", zap.String("message type", reflect.TypeOf(msg.packet).String()[9:]), zap.String("ClientID", c.info.clientID))
+	}
+
 	switch ca.(type) {
 	case *packets.ConnackPacket:
 	case *packets.ConnectPacket:
@@ -161,21 +189,77 @@ func ProcessMessage(msg *Message) {
 }

 func (c *client) ProcessPublish(packet *packets.PublishPacket) {
+	switch c.typ {
+	case CLIENT:
+		c.processClientPublish(packet)
+	case ROUTER:
+		c.processRouterPublish(packet)
+	case CLUSTER:
+		c.processRemotePublish(packet)
+	}
+
+}
+
+func (c *client) processRemotePublish(packet *packets.PublishPacket) {
 	if c.status == Disconnected {
 		return
 	}

 	topic := packet.TopicName
-	if topic == BrokerInfoTopic && c.typ == CLUSTER {
+	if topic == BrokerInfoTopic {
 		c.ProcessInfo(packet)
 		return
 	}

-	if !c.CheckTopicAuth(PUB, topic) {
+}
+
+func (c *client) processRouterPublish(packet *packets.PublishPacket) {
+	if c.status == Disconnected {
+		return
+	}
+
+	switch packet.Qos {
+	case QosAtMostOnce:
+		c.ProcessPublishMessage(packet)
+	case QosAtLeastOnce:
+		puback := packets.NewControlPacket(packets.Puback).(*packets.PubackPacket)
+		puback.MessageID = packet.MessageID
+		if err := c.WriterPacket(puback); err != nil {
+			log.Error("send puback error, ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+			return
+		}
+		c.ProcessPublishMessage(packet)
+	case QosExactlyOnce:
+		return
+	default:
+		log.Error("publish with unknown qos", zap.String("ClientID", c.info.clientID))
+		return
+	}
+
+}
+
+func (c *client) processClientPublish(packet *packets.PublishPacket) {
+	if c.status == Disconnected {
+		return
+	}
+
+	topic := packet.TopicName
+
+	if !c.broker.CheckTopicAuth(PUB, c.info.username, topic) {
 		log.Error("Pub Topics Auth failed, ", zap.String("topic", topic), zap.String("ClientID", c.info.clientID))
 		return
 	}

+	//publish kafka
+	c.broker.Publish(&plugins.Elements{
+		ClientID:  c.info.clientID,
+		Username:  c.info.username,
+		Action:    plugins.Publish,
+		Timestamp: time.Now().Unix(),
+		Payload:   string(packet.Payload),
+		Topic:     topic,
+	})
+
 	switch packet.Qos {
 	case QosAtMostOnce:
 		c.ProcessPublishMessage(packet)
@@ -210,20 +294,19 @@ func (c *client) ProcessPublishMessage(packet *packets.PublishPacket) {
 		}
 	}

-	c.mu.Lock()
 	err := c.topicsMgr.Subscribers([]byte(packet.TopicName), packet.Qos, &c.subs, &c.qoss)
-	c.mu.Unlock()
 	if err != nil {
 		log.Error("Error retrieving subscribers list: ", zap.String("ClientID", c.info.clientID))
 		return
 	}

-	// log.Info("psubs num: ", len(r.psubs))
+	// fmt.Println("psubs num: ", len(c.subs))
 	if len(c.subs) == 0 {
 		return
 	}

-	for _, sub := range c.subs {
+	var qsub []int
+	for i, sub := range c.subs {
 		s, ok := sub.(*subscription)
 		if ok {
 			if s.client.typ == ROUTER {
@@ -231,17 +314,34 @@ func (c *client) ProcessPublishMessage(packet *packets.PublishPacket) {
 					continue
 				}
 			}
-			err := s.client.WriterPacket(packet)
-			if err != nil {
-				log.Error("process message for psub error,  ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+			if s.share {
+				qsub = append(qsub, i)
+			} else {
+				publish(s, packet)
 			}
+
 		}

 	}

+	if len(qsub) > 0 {
+		idx := r.Intn(len(qsub))
+		sub := c.subs[qsub[idx]].(*subscription)
+		publish(sub, packet)
+	}
+
 }

 func (c *client) ProcessSubscribe(packet *packets.SubscribePacket) {
+	switch c.typ {
+	case CLIENT:
+		c.processClientSubscribe(packet)
+	case ROUTER:
+		c.processRouterSubscribe(packet)
+	}
+}
+
+func (c *client) processClientSubscribe(packet *packets.SubscribePacket) {
 	if c.status == Disconnected {
 		return
 	}
@@ -260,25 +360,51 @@ func (c *client) ProcessSubscribe(packet *packets.SubscribePacket) {
 	for i, topic := range topics {
 		t := topic
 		//check topic auth for client
-		if !c.CheckTopicAuth(SUB, topic) {
+		if !b.CheckTopicAuth(SUB, c.info.username, topic) {
 			log.Error("Sub topic Auth failed: ", zap.String("topic", topic), zap.String("ClientID", c.info.clientID))
 			retcodes = append(retcodes, QosFailure)
 			continue
 		}

+		b.Publish(&plugins.Elements{
+			ClientID:  c.info.clientID,
+			Username:  c.info.username,
+			Action:    plugins.Subscribe,
+			Timestamp: time.Now().Unix(),
+			Topic:     topic,
+		})
+
+		groupName := ""
+		share := false
+		if strings.HasPrefix(topic, "$share/") {
+			substr := groupCompile.FindStringSubmatch(topic)
+			if len(substr) != 3 {
+				retcodes = append(retcodes, QosFailure)
+				continue
+			}
+			share = true
+			groupName = substr[1]
+			topic = substr[2]
+		}
+
 		sub := &subscription{
-			topic:  t,
-			qos:    qoss[i],
-			client: c,
+			topic:     topic,
+			qos:       qoss[i],
+			client:    c,
+			share:     share,
+			groupName: groupName,
 		}

 		rqos, err := c.topicsMgr.Subscribe([]byte(topic), qoss[i], sub)
 		if err != nil {
-			return
+			log.Error("subscribe error, ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+			retcodes = append(retcodes, QosFailure)
+			continue
 		}

-		c.subMap[topic] = sub
-		c.session.AddTopic(topic, qoss[i])
+		c.subMap[t] = sub
+
+		c.session.AddTopic(t, qoss[i])
 		retcodes = append(retcodes, rqos)
 		c.topicsMgr.Retained([]byte(topic), &c.rmsgs)

@@ -292,9 +418,7 @@ func (c *client) ProcessSubscribe(packet *packets.SubscribePacket) {
 		return
 	}
 	//broadcast subscribe message
-	if c.typ == CLIENT {
-		go b.BroadcastSubOrUnsubMessage(packet)
-	}
+	go b.BroadcastSubOrUnsubMessage(packet)

 	//process retain message
 	for _, rm := range c.rmsgs {
@@ -306,7 +430,76 @@ func (c *client) ProcessSubscribe(packet *packets.SubscribePacket) {
 	}
 }

+func (c *client) processRouterSubscribe(packet *packets.SubscribePacket) {
+	if c.status == Disconnected {
+		return
+	}
+
+	b := c.broker
+	if b == nil {
+		return
+	}
+	topics := packet.Topics
+	qoss := packet.Qoss
+
+	suback := packets.NewControlPacket(packets.Suback).(*packets.SubackPacket)
+	suback.MessageID = packet.MessageID
+	var retcodes []byte
+
+	for i, topic := range topics {
+		t := topic
+		groupName := ""
+		share := false
+		if strings.HasPrefix(topic, "$share/") {
+			substr := groupCompile.FindStringSubmatch(topic)
+			if len(substr) != 3 {
+				retcodes = append(retcodes, QosFailure)
+				continue
+			}
+			share = true
+			groupName = substr[1]
+			topic = substr[2]
+		}
+
+		sub := &subscription{
+			topic:     topic,
+			qos:       qoss[i],
+			client:    c,
+			share:     share,
+			groupName: groupName,
+		}
+
+		rqos, err := c.topicsMgr.Subscribe([]byte(topic), qoss[i], sub)
+		if err != nil {
+			log.Error("subscribe error, ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+			retcodes = append(retcodes, QosFailure)
+			continue
+		}
+
+		c.subMap[t] = sub
+		addSubMap(c.routeSubMap, topic)
+		retcodes = append(retcodes, rqos)
+	}
+
+	suback.ReturnCodes = retcodes
+
+	err := c.WriterPacket(suback)
+	if err != nil {
+		log.Error("send suback error, ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+		return
+	}
+}
+
 func (c *client) ProcessUnSubscribe(packet *packets.UnsubscribePacket) {
+	switch c.typ {
+	case CLIENT:
+		c.processClientUnSubscribe(packet)
+	case ROUTER:
+		c.processRouterUnSubscribe(packet)
+	}
+}
+
+func (c *client) processRouterUnSubscribe(packet *packets.UnsubscribePacket) {
 	if c.status == Disconnected {
 		return
 	}
@@ -317,13 +510,60 @@ func (c *client) ProcessUnSubscribe(packet *packets.UnsubscribePacket) {
 	topics := packet.Topics

 	for _, topic := range topics {
-		t := []byte(topic)
 		sub, exist := c.subMap[topic]
 		if exist {
-			c.topicsMgr.Unsubscribe(t, sub)
+			retainNum := delSubMap(c.routeSubMap, topic)
+			if retainNum > 0 {
+				continue
+			}
+
+			c.topicsMgr.Unsubscribe([]byte(sub.topic), sub)
+			delete(c.subMap, topic)
+		}
+
+	}
+
+	unsuback := packets.NewControlPacket(packets.Unsuback).(*packets.UnsubackPacket)
+	unsuback.MessageID = packet.MessageID
+
+	err := c.WriterPacket(unsuback)
+	if err != nil {
+		log.Error("send unsuback error, ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+		return
+	}
+}
+
+func (c *client) processClientUnSubscribe(packet *packets.UnsubscribePacket) {
+	if c.status == Disconnected {
+		return
+	}
+	b := c.broker
+	if b == nil {
+		return
+	}
+	topics := packet.Topics
+
+	for _, topic := range topics {
+		{
+			//publish kafka
+
+			b.Publish(&plugins.Elements{
+				ClientID:  c.info.clientID,
+				Username:  c.info.username,
+				Action:    plugins.Unsubscribe,
+				Timestamp: time.Now().Unix(),
+				Topic:     topic,
+			})
+
+		}
+
+		sub, exist := c.subMap[topic]
+		if exist {
+			c.topicsMgr.Unsubscribe([]byte(sub.topic), sub)
 			c.session.RemoveTopic(topic)
 			delete(c.subMap, topic)
 		}
+
 	}

 	unsuback := packets.NewControlPacket(packets.Unsuback).(*packets.UnsubackPacket)
@@ -335,9 +575,7 @@ func (c *client) ProcessUnSubscribe(packet *packets.UnsubscribePacket) {
 		return
 	}
 	// //process ubsubscribe message
-	if c.typ == CLIENT {
-		b.BroadcastSubOrUnsubMessage(packet)
-	}
+	b.BroadcastSubOrUnsubMessage(packet)
 }

 func (c *client) ProcessPing() {
@@ -364,15 +602,29 @@ func (c *client) Close() {
 	// time.Sleep(1 * time.Second)
 	// c.status = Disconnected

+	b := c.broker
+	b.Publish(&plugins.Elements{
+		ClientID:  c.info.clientID,
+		Username:  c.info.username,
+		Action:    plugins.Disconnect,
+		Timestamp: time.Now().Unix(),
+	})
+
 	if c.conn != nil {
 		c.conn.Close()
 		c.conn = nil
 	}

-	b := c.broker
 	subs := c.subMap
+
 	if b != nil {
 		b.removeClient(c)
+		for _, sub := range subs {
+			err := b.topicsMgr.Unsubscribe([]byte(sub.topic), sub)
+			if err != nil {
+				log.Error("unsubscribe error, ", zap.Error(err), zap.String("ClientID", c.info.clientID))
+			}
+		}

 		if c.typ == CLIENT {
 			b.BroadcastUnSubscribe(subs)
@@ -3,13 +3,15 @@
 package broker

 import (
-	"crypto/md5"
-	"crypto/rand"
-	"encoding/base64"
-	"encoding/hex"
-	"io"
+	"encoding/json"
 	"reflect"
 	"time"
+
+	"github.com/tidwall/gjson"
+	"go.uber.org/zap"
+
+	"github.com/eclipse/paho.mqtt.golang/packets"
+	uuid "github.com/satori/go.uuid"
 )

 const (
@@ -91,13 +93,60 @@ func equal(k1, k2 interface{}) bool {
 	return false
 }

+func addSubMap(m map[string]uint64, topic string) {
+	subNum, exist := m[topic]
+	if exist {
+		m[topic] = subNum + 1
+	} else {
+		m[topic] = 1
+	}
+}
+
+func delSubMap(m map[string]uint64, topic string) uint64 {
+	subNum, exist := m[topic]
+	if exist {
+		if subNum > 1 {
+			m[topic] = subNum - 1
+			return subNum - 1
+		}
+	} else {
+		m[topic] = 0
+	}
+	return 0
+}
+
 func GenUniqueId() string {
-	b := make([]byte, 48)
-	if _, err := io.ReadFull(rand.Reader, b); err != nil {
-		return ""
+	return uuid.NewV4().String()
+}
+
+func wrapPublishPacket(packet *packets.PublishPacket) *packets.PublishPacket {
+	p := packet.Copy()
+	wrapPayload := map[string]interface{}{
+		"message_id": GenUniqueId(),
+		"payload":    string(p.Payload),
+	}
+	b, _ := json.Marshal(wrapPayload)
+	p.Payload = b
+	return p
+}
+
+func unWrapPublishPacket(packet *packets.PublishPacket) *packets.PublishPacket {
+	p := packet.Copy()
+	if gjson.GetBytes(p.Payload, "payload").Exists() {
+		p.Payload = []byte(gjson.GetBytes(p.Payload, "payload").String())
+	}
+	return p
+}
+
+func publish(sub *subscription, packet *packets.PublishPacket) {
+	var p *packets.PublishPacket
+	if sub.client.info.username != "root" {
+		p = unWrapPublishPacket(packet)
+	} else {
+		p = wrapPublishPacket(packet)
+	}
+	err := sub.client.WriterPacket(p)
+	if err != nil {
+		log.Error("process message for psub error,  ", zap.Error(err))
 	}
-	h := md5.New()
-	h.Write([]byte(base64.URLEncoding.EncodeToString(b)))
-	return hex.EncodeToString(h.Sum(nil))
-	// return GetMd5String()
 }
@@ -30,7 +30,8 @@ type Config struct {
 	TlsInfo TLSInfo   `json:"tlsInfo"`
 	Acl     bool      `json:"acl"`
 	AclConf string    `json:"aclConf"`
-	Debug   bool      `json:"-"`
+	Debug   bool      `json:"debug"`
+	Plugins []string  `json:"plugins"`
 }

 type RouteInfo struct {
@@ -108,9 +109,6 @@ func ConfigureConfig(args []string) (*Config, error) {
 		}
 	})

-	logger.InitLogger(config.Debug)
-	log = logger.Get().Named("Broker")
-
 	if configFile != "" {
 		tmpConfig, e := LoadConfig(configFile)
 		if e != nil {
@@ -120,6 +118,9 @@ func ConfigureConfig(args []string) (*Config, error) {
 		}
 	}

+	logger.InitLogger(config.Debug)
+	log = logger.Get().Named("Broker")
+
 	if err := config.check(); err != nil {
 		return nil, err
 	}
@@ -132,7 +133,7 @@ func LoadConfig(filename string) (*Config, error) {

 	content, err := ioutil.ReadFile(filename)
 	if err != nil {
-		log.Error("Read config file error: ", zap.Error(err))
+		// log.Error("Read config file error: ", zap.Error(err))
 		return nil, err
 	}
 	// log.Info(string(content))
@@ -140,7 +141,7 @@ func LoadConfig(filename string) (*Config, error) {
 	var config Config
 	err = json.Unmarshal(content, &config)
 	if err != nil {
-		log.Error("Unmarshal config file error: ", zap.Error(err))
+		// log.Error("Unmarshal config file error: ", zap.Error(err))
 		return nil, err
 	}

@@ -0,0 +1,26 @@
+package broker
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+func InitHTTPMoniter(b *Broker) {
+	gin.SetMode(gin.ReleaseMode)
+	router := gin.Default()
+	router.DELETE("api/v1/connections/:clientid", func(c *gin.Context) {
+		clientid := c.Param("clientid")
+		cli, ok := b.clients.Load(clientid)
+		if ok {
+			conn, succss := cli.(*client)
+			if succss {
+				conn.Close()
+			}
+		}
+		resp := map[string]int{
+			"code": 0,
+		}
+		c.JSON(200, &resp)
+	})
+
+	router.Run(":8080")
+}
@@ -0,0 +1,12 @@
+package broker
+
+import (
+	"github.com/fhmq/hmq/plugins"
+	"github.com/fhmq/hmq/plugins/kafka"
+)
+
+func (b *Broker) Publish(e *plugins.Elements) {
+	if b.pluginKafka {
+		kafka.Publish(e)
+	}
+}
@@ -0,0 +1,116 @@
+/* Copyright (c) 2018, joy.zhou <chowyu08@gmail.com>
+ */
+package acl
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	PUB      = 1
+	SUB      = 2
+	PUBSUB   = 3
+	CLIENTID = "clientid"
+	USERNAME = "username"
+	IP       = "ip"
+	ALLOW    = "allow"
+	DENY     = "deny"
+)
+
+type AuthInfo struct {
+	Auth   string
+	Typ    string
+	Val    string
+	PubSub int
+	Topics []string
+}
+
+type ACLConfig struct {
+	File string
+	Info []*AuthInfo
+}
+
+func AclConfigLoad(file string) (*ACLConfig, error) {
+	if file == "" {
+		file = "./conf/acl.conf"
+	}
+	aclconifg := &ACLConfig{
+		File: file,
+		Info: make([]*AuthInfo, 0, 4),
+	}
+	err := aclconifg.Prase()
+	if err != nil {
+		return nil, err
+	}
+	return aclconifg, err
+}
+
+func (c *ACLConfig) Prase() error {
+	f, err := os.Open(c.File)
+	defer f.Close()
+	if err != nil {
+		return err
+	}
+	buf := bufio.NewReader(f)
+	var parseErr error
+	for {
+		line, err := buf.ReadString('\n')
+		line = strings.TrimSpace(line)
+		if isCommentOut(line) {
+			continue
+		}
+		if line == "" {
+			return parseErr
+		}
+		// fmt.Println(line)
+		tmpArr := strings.Fields(line)
+		if len(tmpArr) != 5 {
+			parseErr = errors.New("\"" + line + "\" format is error")
+			break
+		}
+		if tmpArr[0] != ALLOW && tmpArr[0] != DENY {
+			parseErr = errors.New("\"" + line + "\" format is error")
+			break
+		}
+		if tmpArr[1] != CLIENTID && tmpArr[1] != USERNAME && tmpArr[1] != IP {
+			parseErr = errors.New("\"" + line + "\" format is error")
+			break
+		}
+		var pubsub int
+		pubsub, err = strconv.Atoi(tmpArr[3])
+		if err != nil {
+			parseErr = errors.New("\"" + line + "\" format is error")
+			break
+		}
+		topicStr := strings.Replace(tmpArr[4], " ", "", -1)
+		topicStr = strings.Replace(topicStr, "\n", "", -1)
+		topics := strings.Split(topicStr, ",")
+		tmpAuth := &AuthInfo{
+			Auth:   tmpArr[0],
+			Typ:    tmpArr[1],
+			Val:    tmpArr[2],
+			Topics: topics,
+			PubSub: pubsub,
+		}
+		c.Info = append(c.Info, tmpAuth)
+		if err != nil {
+			if err != io.EOF {
+				parseErr = err
+			}
+			break
+		}
+	}
+	return parseErr
+}
+func isCommentOut(line string) bool {
+	if strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "//") || strings.HasPrefix(line, "*") {
+		return true
+	} else {
+		return false
+	}
+}
@@ -0,0 +1,149 @@
+/* Copyright (c) 2018, joy.zhou <chowyu08@gmail.com>*/
+package acl
+
+import "strings"
+
+func CheckTopicAuth(ACLInfo *ACLConfig, typ int, ip, username, clientid, topic string) bool {
+	for _, info := range ACLInfo.Info {
+		ctyp := info.Typ
+		switch ctyp {
+		case CLIENTID:
+			if match, auth := info.checkWithClientID(typ, clientid, topic); match {
+				return auth
+			}
+		case USERNAME:
+			if match, auth := info.checkWithUsername(typ, username, topic); match {
+				return auth
+			}
+		case IP:
+			if match, auth := info.checkWithIP(typ, ip, topic); match {
+				return auth
+			}
+		}
+	}
+	return false
+}
+
+func (a *AuthInfo) checkWithClientID(typ int, clientid, topic string) (bool, bool) {
+	auth := false
+	match := false
+	if a.Val == "*" || a.Val == clientid {
+		for _, tp := range a.Topics {
+			des := strings.Replace(tp, "%c", clientid, -1)
+			if typ == PUB {
+				if pubTopicMatch(topic, des) {
+					match = true
+					auth = a.checkAuth(PUB)
+				}
+			} else if typ == SUB {
+				if subTopicMatch(topic, des) {
+					match = true
+					auth = a.checkAuth(SUB)
+				}
+			}
+		}
+	}
+	return match, auth
+}
+
+func (a *AuthInfo) checkWithUsername(typ int, username, topic string) (bool, bool) {
+	auth := false
+	match := false
+	if a.Val == "*" || a.Val == username {
+		for _, tp := range a.Topics {
+			des := strings.Replace(tp, "%u", username, -1)
+			if typ == PUB {
+				if pubTopicMatch(topic, des) {
+					match = true
+					auth = a.checkAuth(PUB)
+				}
+			} else if typ == SUB {
+				if subTopicMatch(topic, des) {
+					match = true
+					auth = a.checkAuth(SUB)
+				}
+			}
+		}
+	}
+	return match, auth
+}
+
+func (a *AuthInfo) checkWithIP(typ int, ip, topic string) (bool, bool) {
+	auth := false
+	match := false
+	if a.Val == "*" || a.Val == ip {
+		for _, tp := range a.Topics {
+			des := tp
+			if typ == PUB {
+				if pubTopicMatch(topic, des) {
+					auth = a.checkAuth(PUB)
+					match = true
+				}
+			} else if typ == SUB {
+				if subTopicMatch(topic, des) {
+					auth = a.checkAuth(SUB)
+					match = true
+				}
+			}
+		}
+	}
+	return match, auth
+}
+
+func (a *AuthInfo) checkAuth(typ int) bool {
+	auth := false
+	if typ == PUB {
+		if a.Auth == ALLOW && (a.PubSub == PUB || a.PubSub == PUBSUB) {
+			auth = true
+		} else if a.Auth == DENY && a.PubSub == SUB {
+			auth = true
+		}
+	} else if typ == SUB {
+		if a.Auth == ALLOW && (a.PubSub == SUB || a.PubSub == PUBSUB) {
+			auth = true
+		} else if a.Auth == DENY && a.PubSub == PUB {
+			auth = true
+		}
+	}
+	return auth
+}
+
+func pubTopicMatch(pub, des string) bool {
+	dest, _ := SubscribeTopicSpilt(des)
+	topic, _ := PublishTopicSpilt(pub)
+	for i, t := range dest {
+		if i > len(topic)-1 {
+			return false
+		}
+		if t == "#" {
+			return true
+		}
+		if t == "+" || t == topic[i] {
+			continue
+		}
+		if t != topic[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func subTopicMatch(pub, des string) bool {
+	dest, _ := SubscribeTopicSpilt(des)
+	topic, _ := SubscribeTopicSpilt(pub)
+	for i, t := range dest {
+		if i > len(topic)-1 {
+			return false
+		}
+		if t == "#" {
+			return true
+		}
+		if t == "+" || "+" == topic[i] || t == topic[i] {
+			continue
+		}
+		if t != topic[i] {
+			return false
+		}
+	}
+	return true
+}
@@ -0,0 +1,54 @@
+/* Copyright (c) 2018, joy.zhou <chowyu08@gmail.com>
+ */
+package acl
+
+import (
+	"bytes"
+	"errors"
+	"strings"
+)
+
+func SubscribeTopicSpilt(topic string) ([]string, error) {
+	subject := []byte(topic)
+	if bytes.IndexByte(subject, '#') != -1 {
+		if bytes.IndexByte(subject, '#') != len(subject)-1 {
+			return nil, errors.New("Topic format error with index of #")
+		}
+	}
+	re := strings.Split(topic, "/")
+	for i, v := range re {
+		if i != 0 && i != (len(re)-1) {
+			if v == "" {
+				return nil, errors.New("Topic format error with index of //")
+			}
+			if strings.Contains(v, "+") && v != "+" {
+				return nil, errors.New("Topic format error with index of +")
+			}
+		} else {
+			if v == "" {
+				re[i] = "/"
+			}
+		}
+	}
+	return re, nil
+
+}
+
+func PublishTopicSpilt(topic string) ([]string, error) {
+	subject := []byte(topic)
+	if bytes.IndexByte(subject, '#') != -1 || bytes.IndexByte(subject, '+') != -1 {
+		return nil, errors.New("Publish Topic format error with + and #")
+	}
+	re := strings.Split(topic, "/")
+	for i, v := range re {
+		if v == "" {
+			if i != 0 && i != (len(re)-1) {
+				return nil, errors.New("Topic format error with index of //")
+			} else {
+				re[i] = "/"
+			}
+		}
+
+	}
+	return re, nil
+}
@@ -0,0 +1,62 @@
+package sessions
+
+import (
+	"fmt"
+	"sync"
+)
+
+var _ SessionsProvider = (*memProvider)(nil)
+
+func init() {
+	Register("mem", NewMemProvider())
+}
+
+type memProvider struct {
+	st map[string]*Session
+	mu sync.RWMutex
+}
+
+func NewMemProvider() *memProvider {
+	return &memProvider{
+		st: make(map[string]*Session),
+	}
+}
+
+func (this *memProvider) New(id string) (*Session, error) {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	this.st[id] = &Session{id: id}
+	return this.st[id], nil
+}
+
+func (this *memProvider) Get(id string) (*Session, error) {
+	this.mu.RLock()
+	defer this.mu.RUnlock()
+
+	sess, ok := this.st[id]
+	if !ok {
+		return nil, fmt.Errorf("store/Get: No session found for key %s", id)
+	}
+
+	return sess, nil
+}
+
+func (this *memProvider) Del(id string) {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+	delete(this.st, id)
+}
+
+func (this *memProvider) Save(id string) error {
+	return nil
+}
+
+func (this *memProvider) Count() int {
+	return len(this.st)
+}
+
+func (this *memProvider) Close() error {
+	this.st = make(map[string]*Session)
+	return nil
+}
@@ -0,0 +1,149 @@
+package sessions
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/eclipse/paho.mqtt.golang/packets"
+)
+
+const (
+	// Queue size for the ack queue
+	defaultQueueSize = 16
+)
+
+type Session struct {
+
+	// cmsg is the CONNECT message
+	cmsg *packets.ConnectPacket
+
+	// Will message to publish if connect is closed unexpectedly
+	Will *packets.PublishPacket
+
+	// Retained publish message
+	Retained *packets.PublishPacket
+
+	// topics stores all the topis for this session/client
+	topics map[string]byte
+
+	// Initialized?
+	initted bool
+
+	// Serialize access to this session
+	mu sync.Mutex
+
+	id string
+}
+
+func (this *Session) Init(msg *packets.ConnectPacket) error {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	if this.initted {
+		return fmt.Errorf("Session already initialized")
+	}
+
+	this.cmsg = msg
+
+	if this.cmsg.WillFlag {
+		this.Will = packets.NewControlPacket(packets.Publish).(*packets.PublishPacket)
+		this.Will.Qos = this.cmsg.Qos
+		this.Will.TopicName = this.cmsg.WillTopic
+		this.Will.Payload = this.cmsg.WillMessage
+		this.Will.Retain = this.cmsg.WillRetain
+	}
+
+	this.topics = make(map[string]byte, 1)
+
+	this.id = string(msg.ClientIdentifier)
+
+	this.initted = true
+
+	return nil
+}
+
+func (this *Session) Update(msg *packets.ConnectPacket) error {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	this.cmsg = msg
+	return nil
+}
+
+func (this *Session) RetainMessage(msg *packets.PublishPacket) error {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	this.Retained = msg
+
+	return nil
+}
+
+func (this *Session) AddTopic(topic string, qos byte) error {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	if !this.initted {
+		return fmt.Errorf("Session not yet initialized")
+	}
+
+	this.topics[topic] = qos
+
+	return nil
+}
+
+func (this *Session) RemoveTopic(topic string) error {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	if !this.initted {
+		return fmt.Errorf("Session not yet initialized")
+	}
+
+	delete(this.topics, topic)
+
+	return nil
+}
+
+func (this *Session) Topics() ([]string, []byte, error) {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+
+	if !this.initted {
+		return nil, nil, fmt.Errorf("Session not yet initialized")
+	}
+
+	var (
+		topics []string
+		qoss   []byte
+	)
+
+	for k, v := range this.topics {
+		topics = append(topics, k)
+		qoss = append(qoss, v)
+	}
+
+	return topics, qoss, nil
+}
+
+func (this *Session) ID() string {
+	return this.cmsg.ClientIdentifier
+}
+
+func (this *Session) WillFlag() bool {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+	return this.cmsg.WillFlag
+}
+
+func (this *Session) SetWillFlag(v bool) {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+	this.cmsg.WillFlag = v
+}
+
+func (this *Session) CleanSession() bool {
+	this.mu.Lock()
+	defer this.mu.Unlock()
+	return this.cmsg.CleanSession
+}
@@ -0,0 +1,92 @@
+package sessions
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+)
+
+var (
+	ErrSessionsProviderNotFound = errors.New("Session: Session provider not found")
+	ErrKeyNotAvailable          = errors.New("Session: not item found for key.")
+
+	providers = make(map[string]SessionsProvider)
+)
+
+type SessionsProvider interface {
+	New(id string) (*Session, error)
+	Get(id string) (*Session, error)
+	Del(id string)
+	Save(id string) error
+	Count() int
+	Close() error
+}
+
+// Register makes a session provider available by the provided name.
+// If a Register is called twice with the same name or if the driver is nil,
+// it panics.
+func Register(name string, provider SessionsProvider) {
+	if provider == nil {
+		panic("session: Register provide is nil")
+	}
+
+	if _, dup := providers[name]; dup {
+		panic("session: Register called twice for provider " + name)
+	}
+
+	providers[name] = provider
+}
+
+func Unregister(name string) {
+	delete(providers, name)
+}
+
+type Manager struct {
+	p SessionsProvider
+}
+
+func NewManager(providerName string) (*Manager, error) {
+	p, ok := providers[providerName]
+	if !ok {
+		return nil, fmt.Errorf("session: unknown provider %q", providerName)
+	}
+
+	return &Manager{p: p}, nil
+}
+
+func (this *Manager) New(id string) (*Session, error) {
+	if id == "" {
+		id = this.sessionId()
+	}
+	return this.p.New(id)
+}
+
+func (this *Manager) Get(id string) (*Session, error) {
+	return this.p.Get(id)
+}
+
+func (this *Manager) Del(id string) {
+	this.p.Del(id)
+}
+
+func (this *Manager) Save(id string) error {
+	return this.p.Save(id)
+}
+
+func (this *Manager) Count() int {
+	return this.p.Count()
+}
+
+func (this *Manager) Close() error {
+	return this.p.Close()
+}
+
+func (manager *Manager) sessionId() string {
+	b := make([]byte, 15)
+	if _, err := io.ReadFull(rand.Reader, b); err != nil {
+		return ""
+	}
+	return base64.URLEncoding.EncodeToString(b)
+}
@@ -0,0 +1,549 @@
+package topics
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+
+	"github.com/eclipse/paho.mqtt.golang/packets"
+)
+
+const (
+	QosAtMostOnce byte = iota
+	QosAtLeastOnce
+	QosExactlyOnce
+	QosFailure = 0x80
+)
+
+var _ TopicsProvider = (*memTopics)(nil)
+
+type memTopics struct {
+	// Sub/unsub mutex
+	smu sync.RWMutex
+	// Subscription tree
+	sroot *snode
+
+	// Retained message mutex
+	rmu sync.RWMutex
+	// Retained messages topic tree
+	rroot *rnode
+}
+
+func init() {
+	Register("mem", NewMemProvider())
+}
+
+// NewMemProvider returns an new instance of the memTopics, which is implements the
+// TopicsProvider interface. memProvider is a hidden struct that stores the topic
+// subscriptions and retained messages in memory. The content is not persistend so
+// when the server goes, everything will be gone. Use with care.
+func NewMemProvider() *memTopics {
+	return &memTopics{
+		sroot: newSNode(),
+		rroot: newRNode(),
+	}
+}
+
+func ValidQos(qos byte) bool {
+	return qos == QosAtMostOnce || qos == QosAtLeastOnce || qos == QosExactlyOnce
+}
+
+func (this *memTopics) Subscribe(topic []byte, qos byte, sub interface{}) (byte, error) {
+	if !ValidQos(qos) {
+		return QosFailure, fmt.Errorf("Invalid QoS %d", qos)
+	}
+
+	if sub == nil {
+		return QosFailure, fmt.Errorf("Subscriber cannot be nil")
+	}
+
+	this.smu.Lock()
+	defer this.smu.Unlock()
+
+	if qos > QosExactlyOnce {
+		qos = QosExactlyOnce
+	}
+
+	if err := this.sroot.sinsert(topic, qos, sub); err != nil {
+		return QosFailure, err
+	}
+
+	return qos, nil
+}
+
+func (this *memTopics) Unsubscribe(topic []byte, sub interface{}) error {
+	this.smu.Lock()
+	defer this.smu.Unlock()
+
+	return this.sroot.sremove(topic, sub)
+}
+
+// Returned values will be invalidated by the next Subscribers call
+func (this *memTopics) Subscribers(topic []byte, qos byte, subs *[]interface{}, qoss *[]byte) error {
+	if !ValidQos(qos) {
+		return fmt.Errorf("Invalid QoS %d", qos)
+	}
+
+	this.smu.RLock()
+	defer this.smu.RUnlock()
+
+	*subs = (*subs)[0:0]
+	*qoss = (*qoss)[0:0]
+
+	return this.sroot.smatch(topic, qos, subs, qoss)
+}
+
+func (this *memTopics) Retain(msg *packets.PublishPacket) error {
+	this.rmu.Lock()
+	defer this.rmu.Unlock()
+
+	// So apparently, at least according to the MQTT Conformance/Interoperability
+	// Testing, that a payload of 0 means delete the retain message.
+	// https://eclipse.org/paho/clients/testing/
+	if len(msg.Payload) == 0 {
+		return this.rroot.rremove([]byte(msg.TopicName))
+	}
+
+	return this.rroot.rinsert([]byte(msg.TopicName), msg)
+}
+
+func (this *memTopics) Retained(topic []byte, msgs *[]*packets.PublishPacket) error {
+	this.rmu.RLock()
+	defer this.rmu.RUnlock()
+
+	return this.rroot.rmatch(topic, msgs)
+}
+
+func (this *memTopics) Close() error {
+	this.sroot = nil
+	this.rroot = nil
+	return nil
+}
+
+// subscrition nodes
+type snode struct {
+	// If this is the end of the topic string, then add subscribers here
+	subs []interface{}
+	qos  []byte
+
+	// Otherwise add the next topic level here
+	snodes map[string]*snode
+}
+
+func newSNode() *snode {
+	return &snode{
+		snodes: make(map[string]*snode),
+	}
+}
+
+func (this *snode) sinsert(topic []byte, qos byte, sub interface{}) error {
+	// If there's no more topic levels, that means we are at the matching snode
+	// to insert the subscriber. So let's see if there's such subscriber,
+	// if so, update it. Otherwise insert it.
+	if len(topic) == 0 {
+		// Let's see if the subscriber is already on the list. If yes, update
+		// QoS and then return.
+		for i := range this.subs {
+			if equal(this.subs[i], sub) {
+				this.qos[i] = qos
+				return nil
+			}
+		}
+
+		// Otherwise add.
+		this.subs = append(this.subs, sub)
+		this.qos = append(this.qos, qos)
+
+		return nil
+	}
+
+	// Not the last level, so let's find or create the next level snode, and
+	// recursively call it's insert().
+
+	// ntl = next topic level
+	ntl, rem, err := nextTopicLevel(topic)
+	if err != nil {
+		return err
+	}
+
+	level := string(ntl)
+
+	// Add snode if it doesn't already exist
+	n, ok := this.snodes[level]
+	if !ok {
+		n = newSNode()
+		this.snodes[level] = n
+	}
+
+	return n.sinsert(rem, qos, sub)
+}
+
+// This remove implementation ignores the QoS, as long as the subscriber
+// matches then it's removed
+func (this *snode) sremove(topic []byte, sub interface{}) error {
+	// If the topic is empty, it means we are at the final matching snode. If so,
+	// let's find the matching subscribers and remove them.
+	if len(topic) == 0 {
+		// If subscriber == nil, then it's signal to remove ALL subscribers
+		if sub == nil {
+			this.subs = this.subs[0:0]
+			this.qos = this.qos[0:0]
+			return nil
+		}
+
+		// If we find the subscriber then remove it from the list. Technically
+		// we just overwrite the slot by shifting all other items up by one.
+		for i := range this.subs {
+			if equal(this.subs[i], sub) {
+				this.subs = append(this.subs[:i], this.subs[i+1:]...)
+				this.qos = append(this.qos[:i], this.qos[i+1:]...)
+				return nil
+			}
+		}
+
+		return fmt.Errorf("No topic found for subscriber")
+	}
+
+	// Not the last level, so let's find the next level snode, and recursively
+	// call it's remove().
+
+	// ntl = next topic level
+	ntl, rem, err := nextTopicLevel(topic)
+	if err != nil {
+		return err
+	}
+
+	level := string(ntl)
+
+	// Find the snode that matches the topic level
+	n, ok := this.snodes[level]
+	if !ok {
+		return fmt.Errorf("No topic found")
+	}
+
+	// Remove the subscriber from the next level snode
+	if err := n.sremove(rem, sub); err != nil {
+		return err
+	}
+
+	// If there are no more subscribers and snodes to the next level we just visited
+	// let's remove it
+	if len(n.subs) == 0 && len(n.snodes) == 0 {
+		delete(this.snodes, level)
+	}
+
+	return nil
+}
+
+// smatch() returns all the subscribers that are subscribed to the topic. Given a topic
+// with no wildcards (publish topic), it returns a list of subscribers that subscribes
+// to the topic. For each of the level names, it's a match
+// - if there are subscribers to '#', then all the subscribers are added to result set
+func (this *snode) smatch(topic []byte, qos byte, subs *[]interface{}, qoss *[]byte) error {
+	// If the topic is empty, it means we are at the final matching snode. If so,
+	// let's find the subscribers that match the qos and append them to the list.
+	if len(topic) == 0 {
+		this.matchQos(qos, subs, qoss)
+		return nil
+	}
+
+	// ntl = next topic level
+	ntl, rem, err := nextTopicLevel(topic)
+	if err != nil {
+		return err
+	}
+
+	level := string(ntl)
+
+	for k, n := range this.snodes {
+		// If the key is "#", then these subscribers are added to the result set
+		if k == MWC {
+			n.matchQos(qos, subs, qoss)
+		} else if k == SWC || k == level {
+			if err := n.smatch(rem, qos, subs, qoss); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// retained message nodes
+type rnode struct {
+	// If this is the end of the topic string, then add retained messages here
+	msg *packets.PublishPacket
+	// Otherwise add the next topic level here
+	rnodes map[string]*rnode
+}
+
+func newRNode() *rnode {
+	return &rnode{
+		rnodes: make(map[string]*rnode),
+	}
+}
+
+func (this *rnode) rinsert(topic []byte, msg *packets.PublishPacket) error {
+	// If there's no more topic levels, that means we are at the matching rnode.
+	if len(topic) == 0 {
+		// Reuse the message if possible
+		if this.msg == nil {
+			this.msg = msg
+		}
+
+		return nil
+	}
+
+	// Not the last level, so let's find or create the next level snode, and
+	// recursively call it's insert().
+
+	// ntl = next topic level
+	ntl, rem, err := nextTopicLevel(topic)
+	if err != nil {
+		return err
+	}
+
+	level := string(ntl)
+
+	// Add snode if it doesn't already exist
+	n, ok := this.rnodes[level]
+	if !ok {
+		n = newRNode()
+		this.rnodes[level] = n
+	}
+
+	return n.rinsert(rem, msg)
+}
+
+// Remove the retained message for the supplied topic
+func (this *rnode) rremove(topic []byte) error {
+	// If the topic is empty, it means we are at the final matching rnode. If so,
+	// let's remove the buffer and message.
+	if len(topic) == 0 {
+		this.msg = nil
+		return nil
+	}
+
+	// Not the last level, so let's find the next level rnode, and recursively
+	// call it's remove().
+
+	// ntl = next topic level
+	ntl, rem, err := nextTopicLevel(topic)
+	if err != nil {
+		return err
+	}
+
+	level := string(ntl)
+
+	// Find the rnode that matches the topic level
+	n, ok := this.rnodes[level]
+	if !ok {
+		return fmt.Errorf("No topic found")
+	}
+
+	// Remove the subscriber from the next level rnode
+	if err := n.rremove(rem); err != nil {
+		return err
+	}
+
+	// If there are no more rnodes to the next level we just visited let's remove it
+	if len(n.rnodes) == 0 {
+		delete(this.rnodes, level)
+	}
+
+	return nil
+}
+
+// rmatch() finds the retained messages for the topic and qos provided. It's somewhat
+// of a reverse match compare to match() since the supplied topic can contain
+// wildcards, whereas the retained message topic is a full (no wildcard) topic.
+func (this *rnode) rmatch(topic []byte, msgs *[]*packets.PublishPacket) error {
+	// If the topic is empty, it means we are at the final matching rnode. If so,
+	// add the retained msg to the list.
+	if len(topic) == 0 {
+		if this.msg != nil {
+			*msgs = append(*msgs, this.msg)
+		}
+		return nil
+	}
+
+	// ntl = next topic level
+	ntl, rem, err := nextTopicLevel(topic)
+	if err != nil {
+		return err
+	}
+
+	level := string(ntl)
+
+	if level == MWC {
+		// If '#', add all retained messages starting this node
+		this.allRetained(msgs)
+	} else if level == SWC {
+		// If '+', check all nodes at this level. Next levels must be matched.
+		for _, n := range this.rnodes {
+			if err := n.rmatch(rem, msgs); err != nil {
+				return err
+			}
+		}
+	} else {
+		// Otherwise, find the matching node, go to the next level
+		if n, ok := this.rnodes[level]; ok {
+			if err := n.rmatch(rem, msgs); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (this *rnode) allRetained(msgs *[]*packets.PublishPacket) {
+	if this.msg != nil {
+		*msgs = append(*msgs, this.msg)
+	}
+
+	for _, n := range this.rnodes {
+		n.allRetained(msgs)
+	}
+}
+
+const (
+	stateCHR byte = iota // Regular character
+	stateMWC             // Multi-level wildcard
+	stateSWC             // Single-level wildcard
+	stateSEP             // Topic level separator
+	stateSYS             // System level topic ($)
+)
+
+// Returns topic level, remaining topic levels and any errors
+func nextTopicLevel(topic []byte) ([]byte, []byte, error) {
+	s := stateCHR
+
+	for i, c := range topic {
+		switch c {
+		case '/':
+			if s == stateMWC {
+				return nil, nil, fmt.Errorf("Multi-level wildcard found in topic and it's not at the last level")
+			}
+
+			if i == 0 {
+				return []byte(SWC), topic[i+1:], nil
+			}
+
+			return topic[:i], topic[i+1:], nil
+
+		case '#':
+			if i != 0 {
+				return nil, nil, fmt.Errorf("Wildcard character '#' must occupy entire topic level")
+			}
+
+			s = stateMWC
+
+		case '+':
+			if i != 0 {
+				return nil, nil, fmt.Errorf("Wildcard character '+' must occupy entire topic level")
+			}
+
+			s = stateSWC
+
+		// case '$':
+		// 	if i == 0 {
+		// 		return nil, nil, fmt.Errorf("Cannot publish to $ topics")
+		// 	}
+
+		// 	s = stateSYS
+
+		default:
+			if s == stateMWC || s == stateSWC {
+				return nil, nil, fmt.Errorf("Wildcard characters '#' and '+' must occupy entire topic level")
+			}
+
+			s = stateCHR
+		}
+	}
+
+	// If we got here that means we didn't hit the separator along the way, so the
+	// topic is either empty, or does not contain a separator. Either way, we return
+	// the full topic
+	return topic, nil, nil
+}
+
+// The QoS of the payload messages sent in response to a subscription must be the
+// minimum of the QoS of the originally published message (in this case, it's the
+// qos parameter) and the maximum QoS granted by the server (in this case, it's
+// the QoS in the topic tree).
+//
+// It's also possible that even if the topic matches, the subscriber is not included
+// due to the QoS granted is lower than the published message QoS. For example,
+// if the client is granted only QoS 0, and the publish message is QoS 1, then this
+// client is not to be send the published message.
+func (this *snode) matchQos(qos byte, subs *[]interface{}, qoss *[]byte) {
+	for _, sub := range this.subs {
+		// If the published QoS is higher than the subscriber QoS, then we skip the
+		// subscriber. Otherwise, add to the list.
+		// if qos >= this.qos[i] {
+		*subs = append(*subs, sub)
+		*qoss = append(*qoss, qos)
+		// }
+	}
+}
+
+func equal(k1, k2 interface{}) bool {
+	if reflect.TypeOf(k1) != reflect.TypeOf(k2) {
+		return false
+	}
+
+	if reflect.ValueOf(k1).Kind() == reflect.Func {
+		return &k1 == &k2
+	}
+
+	if k1 == k2 {
+		return true
+	}
+
+	switch k1 := k1.(type) {
+	case string:
+		return k1 == k2.(string)
+
+	case int64:
+		return k1 == k2.(int64)
+
+	case int32:
+		return k1 == k2.(int32)
+
+	case int16:
+		return k1 == k2.(int16)
+
+	case int8:
+		return k1 == k2.(int8)
+
+	case int:
+		return k1 == k2.(int)
+
+	case float32:
+		return k1 == k2.(float32)
+
+	case float64:
+		return k1 == k2.(float64)
+
+	case uint:
+		return k1 == k2.(uint)
+
+	case uint8:
+		return k1 == k2.(uint8)
+
+	case uint16:
+		return k1 == k2.(uint16)
+
+	case uint32:
+		return k1 == k2.(uint32)
+
+	case uint64:
+		return k1 == k2.(uint64)
+
+	case uintptr:
+		return k1 == k2.(uintptr)
+	}
+
+	return false
+}
@@ -0,0 +1,91 @@
+package topics
+
+import (
+	"fmt"
+
+	"github.com/eclipse/paho.mqtt.golang/packets"
+)
+
+const (
+	// MWC is the multi-level wildcard
+	MWC = "#"
+
+	// SWC is the single level wildcard
+	SWC = "+"
+
+	// SEP is the topic level separator
+	SEP = "/"
+
+	// SYS is the starting character of the system level topics
+	SYS = "$"
+
+	// Both wildcards
+	_WC = "#+"
+)
+
+var (
+	providers = make(map[string]TopicsProvider)
+)
+
+// TopicsProvider
+type TopicsProvider interface {
+	Subscribe(topic []byte, qos byte, subscriber interface{}) (byte, error)
+	Unsubscribe(topic []byte, subscriber interface{}) error
+	Subscribers(topic []byte, qos byte, subs *[]interface{}, qoss *[]byte) error
+	Retain(msg *packets.PublishPacket) error
+	Retained(topic []byte, msgs *[]*packets.PublishPacket) error
+	Close() error
+}
+
+func Register(name string, provider TopicsProvider) {
+	if provider == nil {
+		panic("topics: Register provide is nil")
+	}
+
+	if _, dup := providers[name]; dup {
+		panic("topics: Register called twice for provider " + name)
+	}
+
+	providers[name] = provider
+}
+
+func Unregister(name string) {
+	delete(providers, name)
+}
+
+type Manager struct {
+	p TopicsProvider
+}
+
+func NewManager(providerName string) (*Manager, error) {
+	p, ok := providers[providerName]
+	if !ok {
+		return nil, fmt.Errorf("session: unknown provider %q", providerName)
+	}
+
+	return &Manager{p: p}, nil
+}
+
+func (this *Manager) Subscribe(topic []byte, qos byte, subscriber interface{}) (byte, error) {
+	return this.p.Subscribe(topic, qos, subscriber)
+}
+
+func (this *Manager) Unsubscribe(topic []byte, subscriber interface{}) error {
+	return this.p.Unsubscribe(topic, subscriber)
+}
+
+func (this *Manager) Subscribers(topic []byte, qos byte, subs *[]interface{}, qoss *[]byte) error {
+	return this.p.Subscribers(topic, qos, subs, qoss)
+}
+
+func (this *Manager) Retain(msg *packets.PublishPacket) error {
+	return this.p.Retain(msg)
+}
+
+func (this *Manager) Retained(topic []byte, msgs *[]*packets.PublishPacket) error {
+	return this.p.Retained(topic, msgs)
+}
+
+func (this *Manager) Close() error {
+	return this.p.Close()
+}