Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.broker.region;
018
019import java.io.IOException;
020import java.util.ArrayList;
021import java.util.LinkedList;
022import java.util.List;
023import java.util.Map;
024import java.util.concurrent.CancellationException;
025import java.util.concurrent.ConcurrentHashMap;
026import java.util.concurrent.ConcurrentMap;
027import java.util.concurrent.CopyOnWriteArrayList;
028import java.util.concurrent.Future;
029import java.util.concurrent.locks.ReentrantReadWriteLock;
030
031import org.apache.activemq.advisory.AdvisorySupport;
032import org.apache.activemq.broker.BrokerService;
033import org.apache.activemq.broker.ConnectionContext;
034import org.apache.activemq.broker.ProducerBrokerExchange;
035import org.apache.activemq.broker.region.policy.DispatchPolicy;
036import org.apache.activemq.broker.region.policy.LastImageSubscriptionRecoveryPolicy;
037import org.apache.activemq.broker.region.policy.RetainedMessageSubscriptionRecoveryPolicy;
038import org.apache.activemq.broker.region.policy.SimpleDispatchPolicy;
039import org.apache.activemq.broker.region.policy.SubscriptionRecoveryPolicy;
040import org.apache.activemq.broker.util.InsertionCountList;
041import org.apache.activemq.command.ActiveMQDestination;
042import org.apache.activemq.command.ConsumerInfo;
043import org.apache.activemq.command.ExceptionResponse;
044import org.apache.activemq.command.Message;
045import org.apache.activemq.command.MessageAck;
046import org.apache.activemq.command.MessageId;
047import org.apache.activemq.command.ProducerAck;
048import org.apache.activemq.command.ProducerInfo;
049import org.apache.activemq.command.Response;
050import org.apache.activemq.command.SubscriptionInfo;
051import org.apache.activemq.filter.MessageEvaluationContext;
052import org.apache.activemq.filter.NonCachedMessageEvaluationContext;
053import org.apache.activemq.store.MessageRecoveryListener;
054import org.apache.activemq.store.NoLocalSubscriptionAware;
055import org.apache.activemq.store.PersistenceAdapter;
056import org.apache.activemq.store.TopicMessageStore;
057import org.apache.activemq.thread.Task;
058import org.apache.activemq.thread.TaskRunner;
059import org.apache.activemq.thread.TaskRunnerFactory;
060import org.apache.activemq.transaction.Synchronization;
061import org.apache.activemq.util.SubscriptionKey;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064
065import javax.jms.JMSException;
066
067import static org.apache.activemq.transaction.Transaction.IN_USE_STATE;
068
069/**
070 * The Topic is a destination that sends a copy of a message to every active
071 * Subscription registered.
072 */
073public class Topic extends BaseDestination implements Task {
074    protected static final Logger LOG = LoggerFactory.getLogger(Topic.class);
075    private final TopicMessageStore topicStore;
076    protected final CopyOnWriteArrayList<Subscription> consumers = new CopyOnWriteArrayList<Subscription>();
077    private final ReentrantReadWriteLock dispatchLock = new ReentrantReadWriteLock();
078    private DispatchPolicy dispatchPolicy = new SimpleDispatchPolicy();
079    private SubscriptionRecoveryPolicy subscriptionRecoveryPolicy;
080    private final ConcurrentMap<SubscriptionKey, DurableTopicSubscription> durableSubscribers = new ConcurrentHashMap<SubscriptionKey, DurableTopicSubscription>();
081    private final TaskRunner taskRunner;
082    private final LinkedList<Runnable> messagesWaitingForSpace = new LinkedList<Runnable>();
083    private final Runnable sendMessagesWaitingForSpaceTask = new Runnable() {
084        @Override
085        public void run() {
086            try {
087                Topic.this.taskRunner.wakeup();
088            } catch (InterruptedException e) {
089            }
090        }
091    };
092
093    public Topic(BrokerService brokerService, ActiveMQDestination destination, TopicMessageStore store,
094            DestinationStatistics parentStats, TaskRunnerFactory taskFactory) throws Exception {
095        super(brokerService, store, destination, parentStats);
096        this.topicStore = store;
097        subscriptionRecoveryPolicy = new RetainedMessageSubscriptionRecoveryPolicy(null);
098        this.taskRunner = taskFactory.createTaskRunner(this, "Topic  " + destination.getPhysicalName());
099    }
100
101    @Override
102    public void initialize() throws Exception {
103        super.initialize();
104        // set non default subscription recovery policy (override policyEntries)
105        if (AdvisorySupport.isMasterBrokerAdvisoryTopic(destination)) {
106            subscriptionRecoveryPolicy = new LastImageSubscriptionRecoveryPolicy();
107            setAlwaysRetroactive(true);
108        }
109        if (store != null) {
110            // AMQ-2586: Better to leave this stat at zero than to give the user
111            // misleading metrics.
112            // int messageCount = store.getMessageCount();
113            // destinationStatistics.getMessages().setCount(messageCount);
114            store.start();
115        }
116    }
117
118    @Override
119    public List<Subscription> getConsumers() {
120        synchronized (consumers) {
121            return new ArrayList<Subscription>(consumers);
122        }
123    }
124
125    public boolean lock(MessageReference node, LockOwner sub) {
126        return true;
127    }
128
129    @Override
130    public void addSubscription(ConnectionContext context, final Subscription sub) throws Exception {
131        if (!sub.getConsumerInfo().isDurable()) {
132
133            // Do a retroactive recovery if needed.
134            if (sub.getConsumerInfo().isRetroactive() || isAlwaysRetroactive()) {
135
136                // synchronize with dispatch method so that no new messages are sent
137                // while we are recovering a subscription to avoid out of order messages.
138                dispatchLock.writeLock().lock();
139                try {
140                    boolean applyRecovery = false;
141                    synchronized (consumers) {
142                        if (!consumers.contains(sub)){
143                            sub.add(context, this);
144                            consumers.add(sub);
145                            applyRecovery=true;
146                            super.addSubscription(context, sub);
147                        }
148                    }
149                    if (applyRecovery){
150                        subscriptionRecoveryPolicy.recover(context, this, sub);
151                    }
152                } finally {
153                    dispatchLock.writeLock().unlock();
154                }
155
156            } else {
157                synchronized (consumers) {
158                    if (!consumers.contains(sub)){
159                        sub.add(context, this);
160                        consumers.add(sub);
161                        super.addSubscription(context, sub);
162                    }
163                }
164            }
165        } else {
166            DurableTopicSubscription dsub = (DurableTopicSubscription) sub;
167            super.addSubscription(context, sub);
168            sub.add(context, this);
169            if(dsub.isActive()) {
170                synchronized (consumers) {
171                    boolean hasSubscription = false;
172
173                    if (consumers.size() == 0) {
174                        hasSubscription = false;
175                    } else {
176                        for (Subscription currentSub : consumers) {
177                            if (currentSub.getConsumerInfo().isDurable()) {
178                                DurableTopicSubscription dcurrentSub = (DurableTopicSubscription) currentSub;
179                                if (dcurrentSub.getSubscriptionKey().equals(dsub.getSubscriptionKey())) {
180                                    hasSubscription = true;
181                                    break;
182                                }
183                            }
184                        }
185                    }
186
187                    if (!hasSubscription) {
188                        consumers.add(sub);
189                    }
190                }
191            }
192            durableSubscribers.put(dsub.getSubscriptionKey(), dsub);
193        }
194    }
195
196    @Override
197    public void removeSubscription(ConnectionContext context, Subscription sub, long lastDeliveredSequenceId) throws Exception {
198        if (!sub.getConsumerInfo().isDurable()) {
199            boolean removed = false;
200            synchronized (consumers) {
201                removed = consumers.remove(sub);
202            }
203            if (removed) {
204                super.removeSubscription(context, sub, lastDeliveredSequenceId);
205            }
206        }
207        sub.remove(context, this);
208    }
209
210    public void deleteSubscription(ConnectionContext context, SubscriptionKey key) throws Exception {
211        if (topicStore != null) {
212            topicStore.deleteSubscription(key.clientId, key.subscriptionName);
213            DurableTopicSubscription removed = durableSubscribers.remove(key);
214            if (removed != null) {
215                destinationStatistics.getConsumers().decrement();
216                // deactivate and remove
217                removed.deactivate(false, 0l);
218                consumers.remove(removed);
219            }
220        }
221    }
222
223    private boolean hasDurableSubChanged(SubscriptionInfo info1, ConsumerInfo info2) throws IOException {
224        if (hasSelectorChanged(info1, info2)) {
225            return true;
226        }
227
228        return hasNoLocalChanged(info1, info2);
229    }
230
231    private boolean hasNoLocalChanged(SubscriptionInfo info1, ConsumerInfo info2) throws IOException {
232        //Not all persistence adapters store the noLocal value for a subscription
233        PersistenceAdapter adapter = broker.getBrokerService().getPersistenceAdapter();
234        if (adapter instanceof NoLocalSubscriptionAware) {
235            if (info1.isNoLocal() ^ info2.isNoLocal()) {
236                return true;
237            }
238        }
239
240        return false;
241    }
242
243    private boolean hasSelectorChanged(SubscriptionInfo info1, ConsumerInfo info2) {
244        if (info1.getSelector() != null ^ info2.getSelector() != null) {
245            return true;
246        }
247
248        if (info1.getSelector() != null && !info1.getSelector().equals(info2.getSelector())) {
249            return true;
250        }
251
252        return false;
253    }
254
255    public void activate(ConnectionContext context, final DurableTopicSubscription subscription) throws Exception {
256        // synchronize with dispatch method so that no new messages are sent
257        // while we are recovering a subscription to avoid out of order messages.
258        dispatchLock.writeLock().lock();
259        try {
260
261            if (topicStore == null) {
262                return;
263            }
264
265            // Recover the durable subscription.
266            String clientId = subscription.getSubscriptionKey().getClientId();
267            String subscriptionName = subscription.getSubscriptionKey().getSubscriptionName();
268            SubscriptionInfo info = topicStore.lookupSubscription(clientId, subscriptionName);
269            if (info != null) {
270                // Check to see if selector changed.
271                if (hasDurableSubChanged(info, subscription.getConsumerInfo())) {
272                    // Need to delete the subscription
273                    topicStore.deleteSubscription(clientId, subscriptionName);
274                    info = null;
275                    // Force a rebuild of the selector chain for the subscription otherwise
276                    // the stored subscription is updated but the selector expression is not
277                    // and the subscription will not behave according to the new configuration.
278                    subscription.setSelector(subscription.getConsumerInfo().getSelector());
279                    synchronized (consumers) {
280                        consumers.remove(subscription);
281                    }
282                } else {
283                    synchronized (consumers) {
284                        if (!consumers.contains(subscription)) {
285                            consumers.add(subscription);
286                        }
287                    }
288                }
289            }
290
291            // Do we need to create the subscription?
292            if (info == null) {
293                info = new SubscriptionInfo();
294                info.setClientId(clientId);
295                info.setSelector(subscription.getConsumerInfo().getSelector());
296                info.setSubscriptionName(subscriptionName);
297                info.setDestination(getActiveMQDestination());
298                info.setNoLocal(subscription.getConsumerInfo().isNoLocal());
299                // This destination is an actual destination id.
300                info.setSubscribedDestination(subscription.getConsumerInfo().getDestination());
301                // This destination might be a pattern
302                synchronized (consumers) {
303                    consumers.add(subscription);
304                    topicStore.addSubscription(info, subscription.getConsumerInfo().isRetroactive());
305                }
306            }
307
308            final MessageEvaluationContext msgContext = new NonCachedMessageEvaluationContext();
309            msgContext.setDestination(destination);
310            if (subscription.isRecoveryRequired()) {
311                topicStore.recoverSubscription(clientId, subscriptionName, new MessageRecoveryListener() {
312                    @Override
313                    public boolean recoverMessage(Message message) throws Exception {
314                        message.setRegionDestination(Topic.this);
315                        try {
316                            msgContext.setMessageReference(message);
317                            if (subscription.matches(message, msgContext)) {
318                                subscription.add(message);
319                            }
320                        } catch (IOException e) {
321                            LOG.error("Failed to recover this message {}", message, e);
322                        }
323                        return true;
324                    }
325
326                    @Override
327                    public boolean recoverMessageReference(MessageId messageReference) throws Exception {
328                        throw new RuntimeException("Should not be called.");
329                    }
330
331                    @Override
332                    public boolean hasSpace() {
333                        return true;
334                    }
335
336                    @Override
337                    public boolean isDuplicate(MessageId id) {
338                        return false;
339                    }
340                });
341            }
342        } finally {
343            dispatchLock.writeLock().unlock();
344        }
345    }
346
347    public void deactivate(ConnectionContext context, DurableTopicSubscription sub, List<MessageReference> dispatched) throws Exception {
348        synchronized (consumers) {
349            consumers.remove(sub);
350        }
351        sub.remove(context, this, dispatched);
352    }
353
354    public void recoverRetroactiveMessages(ConnectionContext context, Subscription subscription) throws Exception {
355        if (subscription.getConsumerInfo().isRetroactive()) {
356            subscriptionRecoveryPolicy.recover(context, this, subscription);
357        }
358    }
359
360    @Override
361    public void send(final ProducerBrokerExchange producerExchange, final Message message) throws Exception {
362        final ConnectionContext context = producerExchange.getConnectionContext();
363
364        final ProducerInfo producerInfo = producerExchange.getProducerState().getInfo();
365        producerExchange.incrementSend();
366        final boolean sendProducerAck = !message.isResponseRequired() && producerInfo.getWindowSize() > 0
367                && !context.isInRecoveryMode();
368
369        message.setRegionDestination(this);
370
371        // There is delay between the client sending it and it arriving at the
372        // destination.. it may have expired.
373        if (message.isExpired()) {
374            broker.messageExpired(context, message, null);
375            getDestinationStatistics().getExpired().increment();
376            if (sendProducerAck) {
377                ProducerAck ack = new ProducerAck(producerInfo.getProducerId(), message.getSize());
378                context.getConnection().dispatchAsync(ack);
379            }
380            return;
381        }
382
383        if (memoryUsage.isFull()) {
384            isFull(context, memoryUsage);
385            fastProducer(context, producerInfo);
386
387            if (isProducerFlowControl() && context.isProducerFlowControl()) {
388
389                if (isFlowControlLogRequired()) {
390                    LOG.info("{}, Usage Manager memory limit reached {}. Producers will be throttled to the rate at which messages are removed from this destination to prevent flooding it. See http://activemq.apache.org/producer-flow-control.html for more info.",
391                            getActiveMQDestination().getQualifiedName(), memoryUsage.getLimit());
392                }
393
394                if (!context.isNetworkConnection() && systemUsage.isSendFailIfNoSpace()) {
395                    throw new javax.jms.ResourceAllocationException("Usage Manager memory limit ("
396                            + memoryUsage.getLimit() + ") reached. Rejecting send for producer (" + message.getProducerId()
397                            + ") to prevent flooding " + getActiveMQDestination().getQualifiedName() + "."
398                            + " See http://activemq.apache.org/producer-flow-control.html for more info");
399                }
400
401                // We can avoid blocking due to low usage if the producer is sending a sync message or
402                // if it is using a producer window
403                if (producerInfo.getWindowSize() > 0 || message.isResponseRequired()) {
404                    synchronized (messagesWaitingForSpace) {
405                        messagesWaitingForSpace.add(new Runnable() {
406                            @Override
407                            public void run() {
408                                try {
409
410                                    // While waiting for space to free up...
411                                    // the transaction may be done
412                                    if (message.isInTransaction()) {
413                                        if (context.getTransaction() == null || context.getTransaction().getState() > IN_USE_STATE) {
414                                            throw new JMSException("Send transaction completed while waiting for space");
415                                        }
416                                    }
417
418                                    // the message may have expired.
419                                    if (message.isExpired()) {
420                                        broker.messageExpired(context, message, null);
421                                        getDestinationStatistics().getExpired().increment();
422                                    } else {
423                                        doMessageSend(producerExchange, message);
424                                    }
425
426                                    if (sendProducerAck) {
427                                        ProducerAck ack = new ProducerAck(producerInfo.getProducerId(), message
428                                                .getSize());
429                                        context.getConnection().dispatchAsync(ack);
430                                    } else {
431                                        Response response = new Response();
432                                        response.setCorrelationId(message.getCommandId());
433                                        context.getConnection().dispatchAsync(response);
434                                    }
435
436                                } catch (Exception e) {
437                                    if (!sendProducerAck && !context.isInRecoveryMode()) {
438                                        ExceptionResponse response = new ExceptionResponse(e);
439                                        response.setCorrelationId(message.getCommandId());
440                                        context.getConnection().dispatchAsync(response);
441                                    }
442                                }
443                            }
444                        });
445
446                        registerCallbackForNotFullNotification();
447                        context.setDontSendReponse(true);
448                        return;
449                    }
450
451                } else {
452                    // Producer flow control cannot be used, so we have do the flow control
453                    // at the broker by blocking this thread until there is space available.
454
455                    if (memoryUsage.isFull()) {
456                        if (context.isInTransaction()) {
457
458                            int count = 0;
459                            while (!memoryUsage.waitForSpace(1000)) {
460                                if (context.getStopping().get()) {
461                                    throw new IOException("Connection closed, send aborted.");
462                                }
463                                if (count > 2 && context.isInTransaction()) {
464                                    count = 0;
465                                    int size = context.getTransaction().size();
466                                    LOG.warn("Waiting for space to send transacted message - transaction elements = {} need more space to commit. Message = {}", size, message);
467                                }
468                                count++;
469                            }
470                        } else {
471                            waitForSpace(
472                                    context,
473                                    producerExchange,
474                                    memoryUsage,
475                                    "Usage Manager Memory Usage limit reached. Stopping producer ("
476                                            + message.getProducerId()
477                                            + ") to prevent flooding "
478                                            + getActiveMQDestination().getQualifiedName()
479                                            + "."
480                                            + " See http://activemq.apache.org/producer-flow-control.html for more info");
481                        }
482                    }
483
484                    // The usage manager could have delayed us by the time
485                    // we unblock the message could have expired..
486                    if (message.isExpired()) {
487                        getDestinationStatistics().getExpired().increment();
488                        LOG.debug("Expired message: {}", message);
489                        return;
490                    }
491                }
492            }
493        }
494
495        doMessageSend(producerExchange, message);
496        messageDelivered(context, message);
497        if (sendProducerAck) {
498            ProducerAck ack = new ProducerAck(producerInfo.getProducerId(), message.getSize());
499            context.getConnection().dispatchAsync(ack);
500        }
501    }
502
503    /**
504     * do send the message - this needs to be synchronized to ensure messages
505     * are stored AND dispatched in the right order
506     *
507     * @param producerExchange
508     * @param message
509     * @throws IOException
510     * @throws Exception
511     */
512    synchronized void doMessageSend(final ProducerBrokerExchange producerExchange, final Message message)
513            throws IOException, Exception {
514        final ConnectionContext context = producerExchange.getConnectionContext();
515        message.getMessageId().setBrokerSequenceId(getDestinationSequenceId());
516        Future<Object> result = null;
517
518        if (topicStore != null && message.isPersistent() && !canOptimizeOutPersistence()) {
519            if (systemUsage.getStoreUsage().isFull(getStoreUsageHighWaterMark())) {
520                final String logMessage = "Persistent store is Full, " + getStoreUsageHighWaterMark() + "% of "
521                        + systemUsage.getStoreUsage().getLimit() + ". Stopping producer (" + message.getProducerId()
522                        + ") to prevent flooding " + getActiveMQDestination().getQualifiedName() + "."
523                        + " See http://activemq.apache.org/producer-flow-control.html for more info";
524                if (!context.isNetworkConnection() && systemUsage.isSendFailIfNoSpace()) {
525                    throw new javax.jms.ResourceAllocationException(logMessage);
526                }
527
528                waitForSpace(context,producerExchange, systemUsage.getStoreUsage(), getStoreUsageHighWaterMark(), logMessage);
529            }
530            result = topicStore.asyncAddTopicMessage(context, message,isOptimizeStorage());
531
532            //Moved the reduceMemoryfootprint clearing to the dispatch method
533        }
534
535        message.incrementReferenceCount();
536
537        if (context.isInTransaction()) {
538            context.getTransaction().addSynchronization(new Synchronization() {
539                @Override
540                public void afterCommit() throws Exception {
541                    // It could take while before we receive the commit
542                    // operation.. by that time the message could have
543                    // expired..
544                    if (message.isExpired()) {
545                        if (broker.isExpired(message)) {
546                            getDestinationStatistics().getExpired().increment();
547                            broker.messageExpired(context, message, null);
548                        }
549                        message.decrementReferenceCount();
550                        return;
551                    }
552                    try {
553                        dispatch(context, message);
554                    } finally {
555                        message.decrementReferenceCount();
556                    }
557                }
558
559                @Override
560                public void afterRollback() throws Exception {
561                    message.decrementReferenceCount();
562                }
563            });
564
565        } else {
566            try {
567                dispatch(context, message);
568            } finally {
569                message.decrementReferenceCount();
570            }
571        }
572
573        if (result != null && !result.isCancelled()) {
574            try {
575                result.get();
576            } catch (CancellationException e) {
577                // ignore - the task has been cancelled if the message
578                // has already been deleted
579            }
580        }
581    }
582
583    private boolean canOptimizeOutPersistence() {
584        return durableSubscribers.size() == 0;
585    }
586
587    @Override
588    public String toString() {
589        return "Topic: destination=" + destination.getPhysicalName() + ", subscriptions=" + consumers.size();
590    }
591
592    @Override
593    public void acknowledge(ConnectionContext context, Subscription sub, final MessageAck ack,
594            final MessageReference node) throws IOException {
595        if (topicStore != null && node.isPersistent()) {
596            DurableTopicSubscription dsub = (DurableTopicSubscription) sub;
597            SubscriptionKey key = dsub.getSubscriptionKey();
598            topicStore.acknowledge(context, key.getClientId(), key.getSubscriptionName(), node.getMessageId(),
599                    convertToNonRangedAck(ack, node));
600        }
601        messageConsumed(context, node);
602    }
603
604    @Override
605    public void gc() {
606    }
607
608    public Message loadMessage(MessageId messageId) throws IOException {
609        return topicStore != null ? topicStore.getMessage(messageId) : null;
610    }
611
612    @Override
613    public void start() throws Exception {
614        if (started.compareAndSet(false, true)) {
615            this.subscriptionRecoveryPolicy.start();
616            if (memoryUsage != null) {
617                memoryUsage.start();
618            }
619
620            if (getExpireMessagesPeriod() > 0 && !AdvisorySupport.isAdvisoryTopic(getActiveMQDestination())) {
621                scheduler.executePeriodically(expireMessagesTask, getExpireMessagesPeriod());
622            }
623        }
624    }
625
626    @Override
627    public void stop() throws Exception {
628        if (started.compareAndSet(true, false)) {
629            if (taskRunner != null) {
630                taskRunner.shutdown();
631            }
632            this.subscriptionRecoveryPolicy.stop();
633            if (memoryUsage != null) {
634                memoryUsage.stop();
635            }
636            if (this.topicStore != null) {
637                this.topicStore.stop();
638            }
639
640            scheduler.cancel(expireMessagesTask);
641        }
642    }
643
644    @Override
645    public Message[] browse() {
646        final List<Message> result = new ArrayList<Message>();
647        doBrowse(result, getMaxBrowsePageSize());
648        return result.toArray(new Message[result.size()]);
649    }
650
651    private void doBrowse(final List<Message> browseList, final int max) {
652        try {
653            if (topicStore != null) {
654                final List<Message> toExpire = new ArrayList<Message>();
655                topicStore.recover(new MessageRecoveryListener() {
656                    @Override
657                    public boolean recoverMessage(Message message) throws Exception {
658                        if (message.isExpired()) {
659                            toExpire.add(message);
660                        }
661                        browseList.add(message);
662                        return true;
663                    }
664
665                    @Override
666                    public boolean recoverMessageReference(MessageId messageReference) throws Exception {
667                        return true;
668                    }
669
670                    @Override
671                    public boolean hasSpace() {
672                        return browseList.size() < max;
673                    }
674
675                    @Override
676                    public boolean isDuplicate(MessageId id) {
677                        return false;
678                    }
679                });
680                final ConnectionContext connectionContext = createConnectionContext();
681                for (Message message : toExpire) {
682                    for (DurableTopicSubscription sub : durableSubscribers.values()) {
683                        if (!sub.isActive()) {
684                            message.setRegionDestination(this);
685                            messageExpired(connectionContext, sub, message);
686                        }
687                    }
688                }
689                Message[] msgs = subscriptionRecoveryPolicy.browse(getActiveMQDestination());
690                if (msgs != null) {
691                    for (int i = 0; i < msgs.length && browseList.size() < max; i++) {
692                        browseList.add(msgs[i]);
693                    }
694                }
695            }
696        } catch (Throwable e) {
697            LOG.warn("Failed to browse Topic: {}", getActiveMQDestination().getPhysicalName(), e);
698        }
699    }
700
701    @Override
702    public boolean iterate() {
703        synchronized (messagesWaitingForSpace) {
704            while (!memoryUsage.isFull() && !messagesWaitingForSpace.isEmpty()) {
705                Runnable op = messagesWaitingForSpace.removeFirst();
706                op.run();
707            }
708
709            if (!messagesWaitingForSpace.isEmpty()) {
710                registerCallbackForNotFullNotification();
711            }
712        }
713        return false;
714    }
715
716    private void registerCallbackForNotFullNotification() {
717        // If the usage manager is not full, then the task will not
718        // get called..
719        if (!memoryUsage.notifyCallbackWhenNotFull(sendMessagesWaitingForSpaceTask)) {
720            // so call it directly here.
721            sendMessagesWaitingForSpaceTask.run();
722        }
723    }
724
725    // Properties
726    // -------------------------------------------------------------------------
727
728    public DispatchPolicy getDispatchPolicy() {
729        return dispatchPolicy;
730    }
731
732    public void setDispatchPolicy(DispatchPolicy dispatchPolicy) {
733        this.dispatchPolicy = dispatchPolicy;
734    }
735
736    public SubscriptionRecoveryPolicy getSubscriptionRecoveryPolicy() {
737        return subscriptionRecoveryPolicy;
738    }
739
740    public void setSubscriptionRecoveryPolicy(SubscriptionRecoveryPolicy recoveryPolicy) {
741        if (this.subscriptionRecoveryPolicy != null && this.subscriptionRecoveryPolicy instanceof RetainedMessageSubscriptionRecoveryPolicy) {
742            // allow users to combine retained message policy with other ActiveMQ policies
743            RetainedMessageSubscriptionRecoveryPolicy policy = (RetainedMessageSubscriptionRecoveryPolicy) this.subscriptionRecoveryPolicy;
744            policy.setWrapped(recoveryPolicy);
745        } else {
746            this.subscriptionRecoveryPolicy = recoveryPolicy;
747        }
748    }
749
750    // Implementation methods
751    // -------------------------------------------------------------------------
752
753    @Override
754    public final void wakeup() {
755    }
756
757    protected void dispatch(final ConnectionContext context, Message message) throws Exception {
758        // AMQ-2586: Better to leave this stat at zero than to give the user
759        // misleading metrics.
760        // destinationStatistics.getMessages().increment();
761        destinationStatistics.getEnqueues().increment();
762        destinationStatistics.getMessageSize().addSize(message.getSize());
763        MessageEvaluationContext msgContext = null;
764
765        dispatchLock.readLock().lock();
766        try {
767            if (!subscriptionRecoveryPolicy.add(context, message)) {
768                return;
769            }
770            synchronized (consumers) {
771                if (consumers.isEmpty()) {
772                    onMessageWithNoConsumers(context, message);
773                    return;
774                }
775            }
776
777            // Clear memory before dispatch - need to clear here because the call to
778            //subscriptionRecoveryPolicy.add() will unmarshall the state
779            if (isReduceMemoryFootprint() && message.isMarshalled()) {
780                message.clearUnMarshalledState();
781            }
782
783            msgContext = context.getMessageEvaluationContext();
784            msgContext.setDestination(destination);
785            msgContext.setMessageReference(message);
786            if (!dispatchPolicy.dispatch(message, msgContext, consumers)) {
787                onMessageWithNoConsumers(context, message);
788            }
789
790        } finally {
791            dispatchLock.readLock().unlock();
792            if (msgContext != null) {
793                msgContext.clear();
794            }
795        }
796    }
797
798    private final Runnable expireMessagesTask = new Runnable() {
799        @Override
800        public void run() {
801            List<Message> browsedMessages = new InsertionCountList<Message>();
802            doBrowse(browsedMessages, getMaxExpirePageSize());
803        }
804    };
805
806    @Override
807    public void messageExpired(ConnectionContext context, Subscription subs, MessageReference reference) {
808        broker.messageExpired(context, reference, subs);
809        // AMQ-2586: Better to leave this stat at zero than to give the user
810        // misleading metrics.
811        // destinationStatistics.getMessages().decrement();
812        destinationStatistics.getExpired().increment();
813        MessageAck ack = new MessageAck();
814        ack.setAckType(MessageAck.STANDARD_ACK_TYPE);
815        ack.setDestination(destination);
816        ack.setMessageID(reference.getMessageId());
817        try {
818            if (subs instanceof DurableTopicSubscription) {
819                ((DurableTopicSubscription)subs).removePending(reference);
820            }
821            acknowledge(context, subs, ack, reference);
822        } catch (Exception e) {
823            LOG.error("Failed to remove expired Message from the store ", e);
824        }
825    }
826
827    @Override
828    protected Logger getLog() {
829        return LOG;
830    }
831
832    protected boolean isOptimizeStorage(){
833        boolean result = false;
834
835        if (isDoOptimzeMessageStorage() && durableSubscribers.isEmpty()==false){
836                result = true;
837                for (DurableTopicSubscription s : durableSubscribers.values()) {
838                    if (s.isActive()== false){
839                        result = false;
840                        break;
841                    }
842                    if (s.getPrefetchSize()==0){
843                        result = false;
844                        break;
845                    }
846                    if (s.isSlowConsumer()){
847                        result = false;
848                        break;
849                    }
850                    if (s.getInFlightUsage() > getOptimizeMessageStoreInFlightLimit()){
851                        result = false;
852                        break;
853                    }
854                }
855        }
856        return result;
857    }
858
859    /**
860     * force a reread of the store - after transaction recovery completion
861     */
862    @Override
863    public void clearPendingMessages() {
864        dispatchLock.readLock().lock();
865        try {
866            for (DurableTopicSubscription durableTopicSubscription : durableSubscribers.values()) {
867                clearPendingAndDispatch(durableTopicSubscription);
868            }
869        } finally {
870            dispatchLock.readLock().unlock();
871        }
872    }
873
874    private void clearPendingAndDispatch(DurableTopicSubscription durableTopicSubscription) {
875        synchronized (durableTopicSubscription.pendingLock) {
876            durableTopicSubscription.pending.clear();
877            try {
878                durableTopicSubscription.dispatchPending();
879            } catch (IOException exception) {
880                LOG.warn("After clear of pending, failed to dispatch to: {}, for: {}, pending: {}, exception: {}", new Object[]{
881                        durableTopicSubscription,
882                        destination,
883                        durableTopicSubscription.pending, exception });
884            }
885        }
886    }
887
888    public Map<SubscriptionKey, DurableTopicSubscription> getDurableTopicSubs() {
889        return durableSubscribers;
890    }
891}